2024-11-07 14:13:40,075 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 14:13:40,125 main DEBUG Took 0.045803 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 14:13:40,126 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 14:13:40,127 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 14:13:40,128 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 14:13:40,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,158 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 14:13:40,184 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,188 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,194 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,197 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,198 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,199 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,200 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,202 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,203 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,206 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,209 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,212 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,213 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,216 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,219 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,225 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,226 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 14:13:40,227 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,228 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 14:13:40,248 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 14:13:40,250 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 14:13:40,254 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 14:13:40,254 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 14:13:40,256 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 14:13:40,257 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 14:13:40,270 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 14:13:40,273 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 14:13:40,276 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 14:13:40,277 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 14:13:40,278 main DEBUG createAppenders(={Console}) 2024-11-07 14:13:40,280 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-07 14:13:40,281 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 14:13:40,282 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-07 14:13:40,284 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 14:13:40,284 main DEBUG OutputStream closed 2024-11-07 14:13:40,286 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 14:13:40,287 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 14:13:40,287 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-07 14:13:40,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 14:13:40,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 14:13:40,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 14:13:40,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 14:13:40,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 14:13:40,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 14:13:40,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 14:13:40,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 14:13:40,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 14:13:40,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 14:13:40,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 14:13:40,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 14:13:40,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 14:13:40,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 14:13:40,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 14:13:40,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 14:13:40,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 14:13:40,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 14:13:40,517 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 14:13:40,517 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-07 14:13:40,518 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 14:13:40,519 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-07T14:13:40,863 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5 2024-11-07 14:13:40,867 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 14:13:40,868 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T14:13:40,880 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-07T14:13:40,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T14:13:40,948 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf, deleteOnExit=true 2024-11-07T14:13:40,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T14:13:40,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/test.cache.data in system properties and HBase conf 2024-11-07T14:13:40,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T14:13:40,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/hadoop.log.dir in system properties and HBase conf 2024-11-07T14:13:40,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T14:13:40,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T14:13:40,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T14:13:41,148 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T14:13:41,278 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T14:13:41,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T14:13:41,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T14:13:41,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T14:13:41,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T14:13:41,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T14:13:41,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T14:13:41,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T14:13:41,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T14:13:41,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T14:13:41,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/nfs.dump.dir in system properties and HBase conf 2024-11-07T14:13:41,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/java.io.tmpdir in system properties and HBase conf 2024-11-07T14:13:41,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T14:13:41,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T14:13:41,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T14:13:42,454 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T14:13:42,537 INFO [Time-limited test {}] log.Log(170): Logging initialized @3546ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T14:13:42,629 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T14:13:42,699 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T14:13:42,726 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T14:13:42,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T14:13:42,729 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T14:13:42,744 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T14:13:42,747 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/hadoop.log.dir/,AVAILABLE} 2024-11-07T14:13:42,748 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T14:13:42,942 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/java.io.tmpdir/jetty-localhost-35329-hadoop-hdfs-3_4_1-tests_jar-_-any-11697250249681481076/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T14:13:42,949 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:35329} 2024-11-07T14:13:42,949 INFO [Time-limited test {}] server.Server(415): Started @3959ms 2024-11-07T14:13:43,505 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T14:13:43,513 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T14:13:43,516 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T14:13:43,516 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T14:13:43,517 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T14:13:43,519 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/hadoop.log.dir/,AVAILABLE} 2024-11-07T14:13:43,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T14:13:43,654 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/java.io.tmpdir/jetty-localhost-40375-hadoop-hdfs-3_4_1-tests_jar-_-any-6191847485109431632/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T14:13:43,655 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:40375} 2024-11-07T14:13:43,656 INFO [Time-limited test {}] server.Server(415): Started @4665ms 2024-11-07T14:13:43,728 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T14:13:44,891 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/data/data1/current/BP-900104924-172.17.0.3-1730988822009/current, will proceed with Du for space computation calculation, 2024-11-07T14:13:44,895 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/data/data2/current/BP-900104924-172.17.0.3-1730988822009/current, will proceed with Du for space computation calculation, 2024-11-07T14:13:44,957 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T14:13:45,011 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x55bef4c9ea80194a with lease ID 0x7487b7b8f9a27f26: Processing first storage report for DS-af1c1953-51ce-465d-acd1-b9b0cc131119 from datanode DatanodeRegistration(127.0.0.1:41543, datanodeUuid=73886870-0704-415d-af4d-78dea2de931a, infoPort=43553, infoSecurePort=0, ipcPort=44885, storageInfo=lv=-57;cid=testClusterID;nsid=930273979;c=1730988822009) 2024-11-07T14:13:45,013 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x55bef4c9ea80194a with lease ID 0x7487b7b8f9a27f26: from storage DS-af1c1953-51ce-465d-acd1-b9b0cc131119 node DatanodeRegistration(127.0.0.1:41543, datanodeUuid=73886870-0704-415d-af4d-78dea2de931a, infoPort=43553, infoSecurePort=0, ipcPort=44885, storageInfo=lv=-57;cid=testClusterID;nsid=930273979;c=1730988822009), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T14:13:45,013 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x55bef4c9ea80194a with lease ID 0x7487b7b8f9a27f26: Processing first storage report for DS-277b4d8e-a6cf-4f38-be6e-41288f0fa567 from datanode DatanodeRegistration(127.0.0.1:41543, datanodeUuid=73886870-0704-415d-af4d-78dea2de931a, infoPort=43553, infoSecurePort=0, ipcPort=44885, storageInfo=lv=-57;cid=testClusterID;nsid=930273979;c=1730988822009) 2024-11-07T14:13:45,014 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x55bef4c9ea80194a with lease ID 0x7487b7b8f9a27f26: from storage DS-277b4d8e-a6cf-4f38-be6e-41288f0fa567 node DatanodeRegistration(127.0.0.1:41543, datanodeUuid=73886870-0704-415d-af4d-78dea2de931a, infoPort=43553, infoSecurePort=0, ipcPort=44885, storageInfo=lv=-57;cid=testClusterID;nsid=930273979;c=1730988822009), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T14:13:45,081 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5 2024-11-07T14:13:45,192 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/zookeeper_0, clientPort=50891, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T14:13:45,205 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50891 2024-11-07T14:13:45,218 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:13:45,222 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:13:45,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741825_1001 (size=7) 2024-11-07T14:13:45,891 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 with version=8 2024-11-07T14:13:45,891 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/hbase-staging 2024-11-07T14:13:46,021 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T14:13:46,346 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b7cc9d764de6:0 server-side Connection retries=45 2024-11-07T14:13:46,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T14:13:46,359 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T14:13:46,365 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T14:13:46,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T14:13:46,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T14:13:46,526 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T14:13:46,599 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T14:13:46,609 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T14:13:46,613 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T14:13:46,637 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1727 (auto-detected) 2024-11-07T14:13:46,638 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-07T14:13:46,658 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40745 2024-11-07T14:13:46,681 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40745 connecting to ZooKeeper ensemble=127.0.0.1:50891 2024-11-07T14:13:46,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:407450x0, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T14:13:46,753 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40745-0x10116d31ff60000 connected 2024-11-07T14:13:46,851 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:13:46,853 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:13:46,862 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T14:13:46,866 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9, hbase.cluster.distributed=false 2024-11-07T14:13:46,895 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T14:13:46,899 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40745 2024-11-07T14:13:46,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40745 2024-11-07T14:13:46,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40745 2024-11-07T14:13:46,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40745 2024-11-07T14:13:46,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40745 2024-11-07T14:13:47,017 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b7cc9d764de6:0 server-side Connection retries=45 2024-11-07T14:13:47,019 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T14:13:47,019 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T14:13:47,019 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T14:13:47,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T14:13:47,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T14:13:47,022 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T14:13:47,025 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T14:13:47,026 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37737 2024-11-07T14:13:47,028 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37737 connecting to ZooKeeper ensemble=127.0.0.1:50891 2024-11-07T14:13:47,029 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:13:47,032 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:13:47,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377370x0, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T14:13:47,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T14:13:47,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37737-0x10116d31ff60001 connected 2024-11-07T14:13:47,051 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T14:13:47,060 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T14:13:47,063 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T14:13:47,069 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T14:13:47,071 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37737 2024-11-07T14:13:47,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37737 2024-11-07T14:13:47,079 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37737 2024-11-07T14:13:47,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37737 2024-11-07T14:13:47,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37737 2024-11-07T14:13:47,101 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b7cc9d764de6:40745 2024-11-07T14:13:47,102 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:47,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T14:13:47,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T14:13:47,118 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:47,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T14:13:47,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:47,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:47,155 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T14:13:47,156 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b7cc9d764de6,40745,1730988826115 from backup master directory 2024-11-07T14:13:47,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T14:13:47,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:47,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T14:13:47,204 WARN [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T14:13:47,205 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:47,207 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T14:13:47,208 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T14:13:47,277 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/hbase.id] with ID: 17ff3ed6-b733-4bf6-b059-c68920fd597e 2024-11-07T14:13:47,277 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/.tmp/hbase.id 2024-11-07T14:13:47,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741826_1002 (size=42) 2024-11-07T14:13:47,295 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/.tmp/hbase.id]:[hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/hbase.id] 2024-11-07T14:13:47,352 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T14:13:47,358 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T14:13:47,380 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-07T14:13:47,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:47,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741827_1003 (size=196) 2024-11-07T14:13:47,453 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:13:47,457 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T14:13:47,478 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:47,484 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T14:13:47,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741828_1004 (size=1189) 2024-11-07T14:13:47,939 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store 2024-11-07T14:13:47,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741829_1005 (size=34) 2024-11-07T14:13:47,965 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T14:13:47,969 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:13:47,970 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T14:13:47,970 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:13:47,971 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:13:47,972 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T14:13:47,972 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:13:47,973 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:13:47,974 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730988827970Disabling compacts and flushes for region at 1730988827970Disabling writes for close at 1730988827972 (+2 ms)Writing region close event to WAL at 1730988827973 (+1 ms)Closed at 1730988827973 2024-11-07T14:13:47,976 WARN [master/b7cc9d764de6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/.initializing 2024-11-07T14:13:47,976 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/WALs/b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:47,987 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T14:13:48,001 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b7cc9d764de6%2C40745%2C1730988826115, suffix=, logDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/WALs/b7cc9d764de6,40745,1730988826115, archiveDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/oldWALs, maxLogs=10 2024-11-07T14:13:48,023 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/WALs/b7cc9d764de6,40745,1730988826115/b7cc9d764de6%2C40745%2C1730988826115.1730988828005, exclude list is [], retry=0 2024-11-07T14:13:48,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41543,DS-af1c1953-51ce-465d-acd1-b9b0cc131119,DISK] 2024-11-07T14:13:48,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T14:13:48,073 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/WALs/b7cc9d764de6,40745,1730988826115/b7cc9d764de6%2C40745%2C1730988826115.1730988828005 2024-11-07T14:13:48,074 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43553:43553)] 2024-11-07T14:13:48,075 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:13:48,075 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:13:48,079 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,079 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,117 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T14:13:48,144 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:48,146 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:48,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,151 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T14:13:48,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:48,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:13:48,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T14:13:48,155 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:48,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:13:48,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T14:13:48,160 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:48,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:13:48,162 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,165 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,167 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,173 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,174 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,178 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T14:13:48,183 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T14:13:48,188 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:13:48,189 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70933829, jitterRate=0.0569964200258255}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T14:13:48,195 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730988828091Initializing all the Stores at 1730988828093 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730988828094 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988828095 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988828095Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988828095Cleaning up temporary data from old regions at 1730988828174 (+79 ms)Region opened successfully at 1730988828195 (+21 ms) 2024-11-07T14:13:48,197 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T14:13:48,230 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2c71b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b7cc9d764de6/172.17.0.3:0 2024-11-07T14:13:48,269 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T14:13:48,284 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T14:13:48,285 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T14:13:48,289 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T14:13:48,291 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T14:13:48,299 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-07T14:13:48,300 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T14:13:48,344 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T14:13:48,355 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T14:13:48,395 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T14:13:48,399 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T14:13:48,401 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T14:13:48,412 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T14:13:48,416 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T14:13:48,420 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T14:13:48,428 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T14:13:48,430 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T14:13:48,440 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T14:13:48,459 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T14:13:48,465 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T14:13:48,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T14:13:48,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T14:13:48,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:48,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:48,482 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b7cc9d764de6,40745,1730988826115, sessionid=0x10116d31ff60000, setting cluster-up flag (Was=false) 2024-11-07T14:13:48,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:48,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:48,537 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T14:13:48,539 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:48,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:48,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:48,582 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T14:13:48,585 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:48,592 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T14:13:48,680 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T14:13:48,689 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(746): ClusterId : 17ff3ed6-b733-4bf6-b059-c68920fd597e 2024-11-07T14:13:48,691 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T14:13:48,692 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T14:13:48,697 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T14:13:48,706 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T14:13:48,706 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T14:13:48,703 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b7cc9d764de6,40745,1730988826115 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T14:13:48,721 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T14:13:48,722 DEBUG [RS:0;b7cc9d764de6:37737 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31607328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b7cc9d764de6/172.17.0.3:0 2024-11-07T14:13:48,723 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b7cc9d764de6:0, corePoolSize=5, maxPoolSize=5 2024-11-07T14:13:48,723 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b7cc9d764de6:0, corePoolSize=5, maxPoolSize=5 2024-11-07T14:13:48,723 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b7cc9d764de6:0, corePoolSize=5, maxPoolSize=5 2024-11-07T14:13:48,723 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b7cc9d764de6:0, corePoolSize=5, maxPoolSize=5 2024-11-07T14:13:48,723 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b7cc9d764de6:0, corePoolSize=10, maxPoolSize=10 2024-11-07T14:13:48,723 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,724 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b7cc9d764de6:0, corePoolSize=2, maxPoolSize=2 2024-11-07T14:13:48,724 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,726 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730988858726 2024-11-07T14:13:48,729 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T14:13:48,730 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T14:13:48,730 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T14:13:48,731 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T14:13:48,735 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T14:13:48,736 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T14:13:48,736 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T14:13:48,737 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T14:13:48,737 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:48,738 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T14:13:48,738 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,742 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b7cc9d764de6:37737 2024-11-07T14:13:48,742 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T14:13:48,743 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T14:13:48,744 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T14:13:48,746 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T14:13:48,746 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T14:13:48,746 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T14:13:48,746 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T14:13:48,747 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T14:13:48,749 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(2659): reportForDuty to master=b7cc9d764de6,40745,1730988826115 with port=37737, startcode=1730988826985 2024-11-07T14:13:48,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741831_1007 (size=1321) 2024-11-07T14:13:48,754 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b7cc9d764de6:0:becomeActiveMaster-HFileCleaner.large.0-1730988828748,5,FailOnTimeoutGroup] 2024-11-07T14:13:48,756 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b7cc9d764de6:0:becomeActiveMaster-HFileCleaner.small.0-1730988828754,5,FailOnTimeoutGroup] 2024-11-07T14:13:48,756 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,756 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T14:13:48,757 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,757 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,760 DEBUG [RS:0;b7cc9d764de6:37737 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T14:13:48,793 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44569, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T14:13:48,798 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40745 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:48,800 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40745 {}] master.ServerManager(517): Registering regionserver=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:48,815 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 2024-11-07T14:13:48,815 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43161 2024-11-07T14:13:48,815 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T14:13:48,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T14:13:48,833 DEBUG [RS:0;b7cc9d764de6:37737 {}] zookeeper.ZKUtil(111): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:48,833 WARN [RS:0;b7cc9d764de6:37737 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T14:13:48,833 INFO [RS:0;b7cc9d764de6:37737 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T14:13:48,833 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/WALs/b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:48,836 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b7cc9d764de6,37737,1730988826985] 2024-11-07T14:13:48,862 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T14:13:48,879 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T14:13:48,884 INFO [RS:0;b7cc9d764de6:37737 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T14:13:48,884 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,886 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T14:13:48,898 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T14:13:48,900 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,900 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,901 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,901 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,901 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,901 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,902 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b7cc9d764de6:0, corePoolSize=2, maxPoolSize=2 2024-11-07T14:13:48,902 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,902 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,902 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,902 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,903 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,903 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b7cc9d764de6:0, corePoolSize=1, maxPoolSize=1 2024-11-07T14:13:48,903 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b7cc9d764de6:0, corePoolSize=3, maxPoolSize=3 2024-11-07T14:13:48,903 DEBUG [RS:0;b7cc9d764de6:37737 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0, corePoolSize=3, maxPoolSize=3 2024-11-07T14:13:48,904 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,905 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,905 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,905 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,905 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,905 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=b7cc9d764de6,37737,1730988826985-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T14:13:48,925 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T14:13:48,927 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=b7cc9d764de6,37737,1730988826985-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,927 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,927 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.Replication(171): b7cc9d764de6,37737,1730988826985 started 2024-11-07T14:13:48,946 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:48,947 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1482): Serving as b7cc9d764de6,37737,1730988826985, RpcServer on b7cc9d764de6/172.17.0.3:37737, sessionid=0x10116d31ff60001 2024-11-07T14:13:48,947 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T14:13:48,948 DEBUG [RS:0;b7cc9d764de6:37737 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:48,948 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b7cc9d764de6,37737,1730988826985' 2024-11-07T14:13:48,948 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T14:13:48,949 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T14:13:48,950 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T14:13:48,950 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T14:13:48,950 DEBUG [RS:0;b7cc9d764de6:37737 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:48,950 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b7cc9d764de6,37737,1730988826985' 2024-11-07T14:13:48,950 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T14:13:48,951 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T14:13:48,952 DEBUG [RS:0;b7cc9d764de6:37737 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T14:13:48,952 INFO [RS:0;b7cc9d764de6:37737 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T14:13:48,952 INFO [RS:0;b7cc9d764de6:37737 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T14:13:49,058 INFO [RS:0;b7cc9d764de6:37737 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T14:13:49,062 INFO [RS:0;b7cc9d764de6:37737 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b7cc9d764de6%2C37737%2C1730988826985, suffix=, logDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/WALs/b7cc9d764de6,37737,1730988826985, archiveDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/oldWALs, maxLogs=32 2024-11-07T14:13:49,081 DEBUG [RS:0;b7cc9d764de6:37737 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/WALs/b7cc9d764de6,37737,1730988826985/b7cc9d764de6%2C37737%2C1730988826985.1730988829065, exclude list is [], retry=0 2024-11-07T14:13:49,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41543,DS-af1c1953-51ce-465d-acd1-b9b0cc131119,DISK] 2024-11-07T14:13:49,098 INFO [RS:0;b7cc9d764de6:37737 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/WALs/b7cc9d764de6,37737,1730988826985/b7cc9d764de6%2C37737%2C1730988826985.1730988829065 2024-11-07T14:13:49,099 DEBUG [RS:0;b7cc9d764de6:37737 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43553:43553)] 2024-11-07T14:13:49,153 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T14:13:49,153 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 2024-11-07T14:13:49,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741833_1009 (size=32) 2024-11-07T14:13:49,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:13:49,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T14:13:49,572 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T14:13:49,572 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:49,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:49,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T14:13:49,577 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T14:13:49,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:49,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:49,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T14:13:49,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T14:13:49,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:49,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:49,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T14:13:49,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T14:13:49,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:49,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:49,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T14:13:49,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740 2024-11-07T14:13:49,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740 2024-11-07T14:13:49,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T14:13:49,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T14:13:49,593 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:13:49,595 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T14:13:49,599 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:13:49,600 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63912880, jitterRate=-0.04762387275695801}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:13:49,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730988829566Initializing all the Stores at 1730988829568 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730988829568Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730988829568Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988829568Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730988829568Cleaning up temporary data from old regions at 1730988829592 (+24 ms)Region opened successfully at 1730988829603 (+11 ms) 2024-11-07T14:13:49,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T14:13:49,603 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T14:13:49,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T14:13:49,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T14:13:49,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T14:13:49,605 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T14:13:49,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730988829603Disabling compacts and flushes for region at 1730988829603Disabling writes for close at 1730988829603Writing region close event to WAL at 1730988829604 (+1 ms)Closed at 1730988829604 2024-11-07T14:13:49,608 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T14:13:49,608 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T14:13:49,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T14:13:49,623 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T14:13:49,626 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T14:13:49,778 DEBUG [b7cc9d764de6:40745 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T14:13:49,789 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:49,795 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b7cc9d764de6,37737,1730988826985, state=OPENING 2024-11-07T14:13:49,828 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T14:13:49,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:49,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:13:49,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T14:13:49,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T14:13:49,839 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T14:13:49,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:13:50,018 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T14:13:50,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33269, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T14:13:50,032 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T14:13:50,032 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T14:13:50,033 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T14:13:50,036 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b7cc9d764de6%2C37737%2C1730988826985.meta, suffix=.meta, logDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/WALs/b7cc9d764de6,37737,1730988826985, archiveDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/oldWALs, maxLogs=32 2024-11-07T14:13:50,051 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/WALs/b7cc9d764de6,37737,1730988826985/b7cc9d764de6%2C37737%2C1730988826985.meta.1730988830038.meta, exclude list is [], retry=0 2024-11-07T14:13:50,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41543,DS-af1c1953-51ce-465d-acd1-b9b0cc131119,DISK] 2024-11-07T14:13:50,058 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/WALs/b7cc9d764de6,37737,1730988826985/b7cc9d764de6%2C37737%2C1730988826985.meta.1730988830038.meta 2024-11-07T14:13:50,058 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43553:43553)] 2024-11-07T14:13:50,058 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:13:50,060 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T14:13:50,063 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T14:13:50,068 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T14:13:50,073 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T14:13:50,074 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:13:50,074 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T14:13:50,074 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T14:13:50,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T14:13:50,079 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T14:13:50,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:50,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:50,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T14:13:50,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T14:13:50,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:50,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:50,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T14:13:50,085 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T14:13:50,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:50,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:50,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T14:13:50,087 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T14:13:50,087 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:50,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T14:13:50,089 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T14:13:50,090 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740 2024-11-07T14:13:50,093 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740 2024-11-07T14:13:50,096 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T14:13:50,096 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T14:13:50,097 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:13:50,100 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T14:13:50,102 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72257175, jitterRate=0.07671581208705902}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:13:50,102 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T14:13:50,104 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730988830075Writing region info on filesystem at 1730988830075Initializing all the Stores at 1730988830077 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730988830077Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730988830078 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988830078Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730988830078Cleaning up temporary data from old regions at 1730988830096 (+18 ms)Running coprocessor post-open hooks at 1730988830102 (+6 ms)Region opened successfully at 1730988830104 (+2 ms) 2024-11-07T14:13:50,112 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730988830010 2024-11-07T14:13:50,123 DEBUG [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T14:13:50,124 INFO [RS_OPEN_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T14:13:50,125 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:50,128 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b7cc9d764de6,37737,1730988826985, state=OPEN 2024-11-07T14:13:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T14:13:50,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T14:13:50,193 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T14:13:50,193 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T14:13:50,194 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:50,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T14:13:50,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b7cc9d764de6,37737,1730988826985 in 353 msec 2024-11-07T14:13:50,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T14:13:50,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 588 msec 2024-11-07T14:13:50,210 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T14:13:50,210 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T14:13:50,228 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:50,229 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:50,247 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:50,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35135, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:50,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6450 sec 2024-11-07T14:13:50,292 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730988830291, completionTime=-1 2024-11-07T14:13:50,294 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T14:13:50,295 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T14:13:50,327 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-07T14:13:50,327 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730988890327 2024-11-07T14:13:50,327 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730988950327 2024-11-07T14:13:50,327 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 32 msec 2024-11-07T14:13:50,330 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b7cc9d764de6,40745,1730988826115-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:50,330 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b7cc9d764de6,40745,1730988826115-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:50,330 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b7cc9d764de6,40745,1730988826115-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:50,332 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b7cc9d764de6:40745, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:50,333 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:50,333 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:50,339 DEBUG [master/b7cc9d764de6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T14:13:50,359 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.154sec 2024-11-07T14:13:50,360 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T14:13:50,361 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T14:13:50,362 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T14:13:50,362 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T14:13:50,363 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T14:13:50,363 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b7cc9d764de6,40745,1730988826115-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T14:13:50,363 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b7cc9d764de6,40745,1730988826115-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T14:13:50,372 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T14:13:50,373 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T14:13:50,374 INFO [master/b7cc9d764de6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b7cc9d764de6,40745,1730988826115-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T14:13:50,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:50,403 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T14:13:50,403 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T14:13:50,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:50,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:50,415 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ecb092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:50,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:50,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:50,429 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:50,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:50,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:50,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:50,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=615, ProcessCount=11, AvailableMemoryMB=257 2024-11-07T14:13:50,484 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T14:13:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13ee3c32 2024-11-07T14:13:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T14:13:50,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T14:13:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:13:50,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:13:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T14:13:50,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:13:50,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-07T14:13:50,540 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:50,543 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:13:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T14:13:50,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741835_1011 (size=959) 2024-11-07T14:13:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T14:13:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T14:13:50,978 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 2024-11-07T14:13:50,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741836_1012 (size=53) 2024-11-07T14:13:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T14:13:51,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:13:51,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8133e138f01b5d4b631e25ead6cfe8cd, disabling compactions & flushes 2024-11-07T14:13:51,392 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:51,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:51,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. after waiting 0 ms 2024-11-07T14:13:51,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:51,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:51,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8133e138f01b5d4b631e25ead6cfe8cd: Waiting for close lock at 1730988831392Disabling compacts and flushes for region at 1730988831392Disabling writes for close at 1730988831393 (+1 ms)Writing region close event to WAL at 1730988831393Closed at 1730988831393 2024-11-07T14:13:51,398 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:13:51,407 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730988831398"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730988831398"}]},"ts":"1730988831398"} 2024-11-07T14:13:51,412 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T14:13:51,414 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:13:51,418 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988831415"}]},"ts":"1730988831415"} 2024-11-07T14:13:51,426 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T14:13:51,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, ASSIGN}] 2024-11-07T14:13:51,431 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, ASSIGN 2024-11-07T14:13:51,432 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, ASSIGN; state=OFFLINE, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=false 2024-11-07T14:13:51,584 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8133e138f01b5d4b631e25ead6cfe8cd, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:51,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, ASSIGN because future has completed 2024-11-07T14:13:51,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:13:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T14:13:51,753 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:51,754 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:13:51,754 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,754 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:13:51,754 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,754 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,757 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,760 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8133e138f01b5d4b631e25ead6cfe8cd columnFamilyName A 2024-11-07T14:13:51,760 DEBUG [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:51,761 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] regionserver.HStore(327): Store=8133e138f01b5d4b631e25ead6cfe8cd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:13:51,762 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,764 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8133e138f01b5d4b631e25ead6cfe8cd columnFamilyName B 2024-11-07T14:13:51,765 DEBUG [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:51,766 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] regionserver.HStore(327): Store=8133e138f01b5d4b631e25ead6cfe8cd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:13:51,766 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,769 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8133e138f01b5d4b631e25ead6cfe8cd columnFamilyName C 2024-11-07T14:13:51,769 DEBUG [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:13:51,770 INFO [StoreOpener-8133e138f01b5d4b631e25ead6cfe8cd-1 {}] regionserver.HStore(327): Store=8133e138f01b5d4b631e25ead6cfe8cd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:13:51,770 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,772 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,773 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,775 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,775 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,776 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:13:51,778 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,781 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:13:51,782 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8133e138f01b5d4b631e25ead6cfe8cd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66100886, jitterRate=-0.015020042657852173}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:13:51,782 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:51,783 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8133e138f01b5d4b631e25ead6cfe8cd: Running coprocessor pre-open hook at 1730988831755Writing region info on filesystem at 1730988831755Initializing all the Stores at 1730988831757 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988831757Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988831757Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988831757Cleaning up temporary data from old regions at 1730988831775 (+18 ms)Running coprocessor post-open hooks at 1730988831782 (+7 ms)Region opened successfully at 1730988831783 (+1 ms) 2024-11-07T14:13:51,785 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., pid=6, masterSystemTime=1730988831745 2024-11-07T14:13:51,789 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:51,789 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:51,790 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8133e138f01b5d4b631e25ead6cfe8cd, regionState=OPEN, openSeqNum=2, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:51,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:13:51,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T14:13:51,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 in 207 msec 2024-11-07T14:13:51,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T14:13:51,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, ASSIGN in 372 msec 2024-11-07T14:13:51,806 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:13:51,806 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988831806"}]},"ts":"1730988831806"} 2024-11-07T14:13:51,810 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T14:13:51,811 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:13:51,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2840 sec 2024-11-07T14:13:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T14:13:52,714 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:13:52,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447f8921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,726 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20f617ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,729 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,737 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T14:13:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b7cc9d764de6,40745,1730988826115 2024-11-07T14:13:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1dd88882 2024-11-07T14:13:52,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T14:13:52,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T14:13:52,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657a16e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,755 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@828ad61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,762 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@374defb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,766 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678699a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da37544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,778 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f953883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,782 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2311693d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a82094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,789 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@831686b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,792 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43df1407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,797 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15646188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,800 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638af609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,804 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dfc701c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,806 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166b4003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,810 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e981b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2268467d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:13:52,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:13:52,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:13:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:13:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:13:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70bfcb87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:13:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:13:52,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:13:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dba7216, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:13:52,842 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,842 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:13:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,848 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,850 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,850 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-07T14:13:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T14:13:52,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,856 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:13:52,857 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,861 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:13:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:13:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,870 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,872 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,889 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:13:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:13:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:13:52,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:13:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:13:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:52,937 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:13:52,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:13:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T14:13:53,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:53,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/d70d14f7faae43c6bad178bcecf0bd3d is 50, key is test_row_0/A:col10/1730988832939/Put/seqid=0 2024-11-07T14:13:53,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741837_1013 (size=14341) 2024-11-07T14:13:53,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988893127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988893128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988893132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988893137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988893163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T14:13:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:53,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:53,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988893304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988893304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988893305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988893305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988893313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:53,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/d70d14f7faae43c6bad178bcecf0bd3d 2024-11-07T14:13:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T14:13:53,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988893523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988893525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988893525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988893526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988893525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:53,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:53,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/3b657b4544b444a682ca9c97e0a8e52d is 50, key is test_row_0/B:col10/1730988832939/Put/seqid=0 2024-11-07T14:13:53,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741838_1014 (size=12001) 2024-11-07T14:13:53,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/3b657b4544b444a682ca9c97e0a8e52d 2024-11-07T14:13:53,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/e436103af5384fdab858395677e2daeb is 50, key is test_row_0/C:col10/1730988832939/Put/seqid=0 2024-11-07T14:13:53,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741839_1015 (size=12001) 2024-11-07T14:13:53,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/e436103af5384fdab858395677e2daeb 2024-11-07T14:13:53,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/d70d14f7faae43c6bad178bcecf0bd3d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d70d14f7faae43c6bad178bcecf0bd3d 2024-11-07T14:13:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d70d14f7faae43c6bad178bcecf0bd3d, entries=200, sequenceid=14, filesize=14.0 K 2024-11-07T14:13:53,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/3b657b4544b444a682ca9c97e0a8e52d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3b657b4544b444a682ca9c97e0a8e52d 2024-11-07T14:13:53,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3b657b4544b444a682ca9c97e0a8e52d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T14:13:53,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/e436103af5384fdab858395677e2daeb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e436103af5384fdab858395677e2daeb 2024-11-07T14:13:53,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e436103af5384fdab858395677e2daeb, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T14:13:53,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8133e138f01b5d4b631e25ead6cfe8cd in 848ms, sequenceid=14, compaction requested=false 2024-11-07T14:13:53,793 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T14:13:53,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:53,846 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:13:53,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:13:53,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:53,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:53,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:53,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/479da619c7b340e798052fcf88d16033 is 50, key is test_row_0/A:col10/1730988833101/Put/seqid=0 2024-11-07T14:13:53,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741840_1016 (size=14341) 2024-11-07T14:13:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988893873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988893877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988893877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988893881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988893884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:53,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988893993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T14:13:54,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988894003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988894005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988894005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988894006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988894203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988894222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988894223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988894223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988894224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/479da619c7b340e798052fcf88d16033 2024-11-07T14:13:54,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/a823d02ada1443d18885d00750ddcbe3 is 50, key is test_row_0/B:col10/1730988833101/Put/seqid=0 2024-11-07T14:13:54,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741841_1017 (size=12001) 2024-11-07T14:13:54,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/a823d02ada1443d18885d00750ddcbe3 2024-11-07T14:13:54,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:54,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:54,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c44df8fa5fdb4d64873bdf5b126dc2a7 is 50, key is test_row_0/C:col10/1730988833101/Put/seqid=0 2024-11-07T14:13:54,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741842_1018 (size=12001) 2024-11-07T14:13:54,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988894513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988894543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988894552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988894553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988894554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:54,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c44df8fa5fdb4d64873bdf5b126dc2a7 2024-11-07T14:13:54,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/479da619c7b340e798052fcf88d16033 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/479da619c7b340e798052fcf88d16033 2024-11-07T14:13:54,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:54,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:54,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:54,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/479da619c7b340e798052fcf88d16033, entries=200, sequenceid=38, filesize=14.0 K 2024-11-07T14:13:54,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/a823d02ada1443d18885d00750ddcbe3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/a823d02ada1443d18885d00750ddcbe3 2024-11-07T14:13:54,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/a823d02ada1443d18885d00750ddcbe3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T14:13:54,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c44df8fa5fdb4d64873bdf5b126dc2a7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c44df8fa5fdb4d64873bdf5b126dc2a7 2024-11-07T14:13:54,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c44df8fa5fdb4d64873bdf5b126dc2a7, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T14:13:54,856 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T14:13:54,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8133e138f01b5d4b631e25ead6cfe8cd in 1012ms, sequenceid=38, compaction requested=false 2024-11-07T14:13:54,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:54,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T14:13:54,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:54,945 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:13:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/c7ee26285e3141c5a716c977346f35b6 is 50, key is test_row_0/A:col10/1730988833851/Put/seqid=0 2024-11-07T14:13:54,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741843_1019 (size=12001) 2024-11-07T14:13:54,995 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/c7ee26285e3141c5a716c977346f35b6 2024-11-07T14:13:55,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/754601f4616d4d279e467bafe47c51a7 is 50, key is test_row_0/B:col10/1730988833851/Put/seqid=0 2024-11-07T14:13:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T14:13:55,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741844_1020 (size=12001) 2024-11-07T14:13:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:55,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:55,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988895144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988895150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988895153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988895156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988895156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988895263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988895264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988895273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988895275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988895276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,429 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/754601f4616d4d279e467bafe47c51a7 2024-11-07T14:13:55,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6685dbd7aff5478ea982e7cc622d1b86 is 50, key is test_row_0/C:col10/1730988833851/Put/seqid=0 2024-11-07T14:13:55,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741845_1021 (size=12001) 2024-11-07T14:13:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988895474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988895483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988895493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988895494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988895494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,585 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:13:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988895784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988895793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988895802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988895815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988895813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:55,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6685dbd7aff5478ea982e7cc622d1b86 2024-11-07T14:13:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/c7ee26285e3141c5a716c977346f35b6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c7ee26285e3141c5a716c977346f35b6 2024-11-07T14:13:55,911 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c7ee26285e3141c5a716c977346f35b6, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T14:13:55,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/754601f4616d4d279e467bafe47c51a7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/754601f4616d4d279e467bafe47c51a7 2024-11-07T14:13:55,955 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/754601f4616d4d279e467bafe47c51a7, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T14:13:55,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6685dbd7aff5478ea982e7cc622d1b86 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6685dbd7aff5478ea982e7cc622d1b86 2024-11-07T14:13:56,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6685dbd7aff5478ea982e7cc622d1b86, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T14:13:56,010 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8133e138f01b5d4b631e25ead6cfe8cd in 1064ms, sequenceid=50, compaction requested=true 2024-11-07T14:13:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-07T14:13:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-07T14:13:56,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T14:13:56,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1520 sec 2024-11-07T14:13:56,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.1770 sec 2024-11-07T14:13:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:56,312 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:13:56,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T14:13:56,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7fb0ee9125a24baf9c90f90b0918a2f7 is 50, key is test_row_0/A:col10/1730988836306/Put/seqid=0 2024-11-07T14:13:56,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988896341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988896341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988896346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988896347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988896351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741846_1022 (size=12001) 2024-11-07T14:13:56,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988896463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988896464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988896464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988896473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,595 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T14:13:56,596 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T14:13:56,600 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T14:13:56,600 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T14:13:56,600 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T14:13:56,600 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T14:13:56,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T14:13:56,602 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T14:13:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988896673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988896682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988896682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988896682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7fb0ee9125a24baf9c90f90b0918a2f7 2024-11-07T14:13:56,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6138ee70655e491da173dc5a15e16942 is 50, key is test_row_0/B:col10/1730988836306/Put/seqid=0 2024-11-07T14:13:56,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741847_1023 (size=12001) 2024-11-07T14:13:56,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6138ee70655e491da173dc5a15e16942 2024-11-07T14:13:56,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/b5736d3f07184cffb1dfb3bacacf9c31 is 50, key is test_row_0/C:col10/1730988836306/Put/seqid=0 2024-11-07T14:13:56,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741848_1024 (size=12001) 2024-11-07T14:13:56,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/b5736d3f07184cffb1dfb3bacacf9c31 2024-11-07T14:13:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7fb0ee9125a24baf9c90f90b0918a2f7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fb0ee9125a24baf9c90f90b0918a2f7 2024-11-07T14:13:56,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988896982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:56,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988896993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:56,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fb0ee9125a24baf9c90f90b0918a2f7, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T14:13:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6138ee70655e491da173dc5a15e16942 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6138ee70655e491da173dc5a15e16942 2024-11-07T14:13:57,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988897003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988897003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6138ee70655e491da173dc5a15e16942, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T14:13:57,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/b5736d3f07184cffb1dfb3bacacf9c31 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b5736d3f07184cffb1dfb3bacacf9c31 2024-11-07T14:13:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T14:13:57,032 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:13:57,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:13:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-07T14:13:57,038 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:13:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T14:13:57,040 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:13:57,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:13:57,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b5736d3f07184cffb1dfb3bacacf9c31, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T14:13:57,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 8133e138f01b5d4b631e25ead6cfe8cd in 734ms, sequenceid=77, compaction requested=true 2024-11-07T14:13:57,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:57,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:13:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:13:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:13:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:13:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:13:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:13:57,055 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:13:57,062 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:13:57,062 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:13:57,064 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:13:57,065 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:57,066 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3b657b4544b444a682ca9c97e0a8e52d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/a823d02ada1443d18885d00750ddcbe3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/754601f4616d4d279e467bafe47c51a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6138ee70655e491da173dc5a15e16942] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=46.9 K 2024-11-07T14:13:57,068 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b657b4544b444a682ca9c97e0a8e52d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988832875 2024-11-07T14:13:57,069 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting a823d02ada1443d18885d00750ddcbe3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730988833101 2024-11-07T14:13:57,070 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 754601f4616d4d279e467bafe47c51a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730988833851 2024-11-07T14:13:57,072 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6138ee70655e491da173dc5a15e16942, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730988835152 2024-11-07T14:13:57,072 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:13:57,073 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:13:57,073 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:57,073 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d70d14f7faae43c6bad178bcecf0bd3d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/479da619c7b340e798052fcf88d16033, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c7ee26285e3141c5a716c977346f35b6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fb0ee9125a24baf9c90f90b0918a2f7] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=51.4 K 2024-11-07T14:13:57,074 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d70d14f7faae43c6bad178bcecf0bd3d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988832875 2024-11-07T14:13:57,076 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479da619c7b340e798052fcf88d16033, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730988833101 2024-11-07T14:13:57,077 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7ee26285e3141c5a716c977346f35b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730988833851 2024-11-07T14:13:57,079 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fb0ee9125a24baf9c90f90b0918a2f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730988835152 2024-11-07T14:13:57,141 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#12 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:13:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T14:13:57,143 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/81069e70ac2f4785b58ce5332c53717c is 50, key is test_row_0/B:col10/1730988836306/Put/seqid=0 2024-11-07T14:13:57,154 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#13 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:13:57,156 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/2e898e260aae43639ca11646552629f9 is 50, key is test_row_0/A:col10/1730988836306/Put/seqid=0 2024-11-07T14:13:57,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741849_1025 (size=12139) 2024-11-07T14:13:57,187 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/81069e70ac2f4785b58ce5332c53717c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/81069e70ac2f4785b58ce5332c53717c 2024-11-07T14:13:57,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T14:13:57,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:57,196 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T14:13:57,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741850_1026 (size=12139) 2024-11-07T14:13:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/209448dd382b4dbaba27d18c6e8010c2 is 50, key is test_row_0/A:col10/1730988836341/Put/seqid=0 2024-11-07T14:13:57,221 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/2e898e260aae43639ca11646552629f9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2e898e260aae43639ca11646552629f9 2024-11-07T14:13:57,228 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into 81069e70ac2f4785b58ce5332c53717c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:13:57,228 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:57,233 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=12, startTime=1730988837052; duration=0sec 2024-11-07T14:13:57,237 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:13:57,238 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:13:57,238 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:13:57,244 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:13:57,244 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:13:57,244 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:57,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741851_1027 (size=12001) 2024-11-07T14:13:57,245 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e436103af5384fdab858395677e2daeb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c44df8fa5fdb4d64873bdf5b126dc2a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6685dbd7aff5478ea982e7cc622d1b86, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b5736d3f07184cffb1dfb3bacacf9c31] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=46.9 K 2024-11-07T14:13:57,246 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e436103af5384fdab858395677e2daeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988832875 2024-11-07T14:13:57,248 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c44df8fa5fdb4d64873bdf5b126dc2a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730988833101 2024-11-07T14:13:57,249 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/209448dd382b4dbaba27d18c6e8010c2 2024-11-07T14:13:57,251 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 2e898e260aae43639ca11646552629f9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:13:57,251 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:57,251 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=12, startTime=1730988837048; duration=0sec 2024-11-07T14:13:57,252 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6685dbd7aff5478ea982e7cc622d1b86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730988833851 2024-11-07T14:13:57,252 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:13:57,252 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:13:57,257 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b5736d3f07184cffb1dfb3bacacf9c31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730988835152 2024-11-07T14:13:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/df5f05275692478ab7ddedb50d18f2c8 is 50, key is test_row_0/B:col10/1730988836341/Put/seqid=0 2024-11-07T14:13:57,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741852_1028 (size=12001) 2024-11-07T14:13:57,299 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#16 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:13:57,300 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/5dfe4826a3dd4a619d9cf3f4f5ea667f is 50, key is test_row_0/C:col10/1730988836306/Put/seqid=0 2024-11-07T14:13:57,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741853_1029 (size=12139) 2024-11-07T14:13:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T14:13:57,357 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/5dfe4826a3dd4a619d9cf3f4f5ea667f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/5dfe4826a3dd4a619d9cf3f4f5ea667f 2024-11-07T14:13:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:57,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:57,376 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into 5dfe4826a3dd4a619d9cf3f4f5ea667f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:13:57,377 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:57,377 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=12, startTime=1730988837052; duration=0sec 2024-11-07T14:13:57,377 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:13:57,377 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:13:57,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988897523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988897523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988897523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988897526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988897533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988897642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988897642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988897643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988897643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T14:13:57,685 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/df5f05275692478ab7ddedb50d18f2c8 2024-11-07T14:13:57,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/8bd3810980d94a139a0253f860934967 is 50, key is test_row_0/C:col10/1730988836341/Put/seqid=0 2024-11-07T14:13:57,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741854_1030 (size=12001) 2024-11-07T14:13:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988897852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988897852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988897853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988897853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,134 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/8bd3810980d94a139a0253f860934967 2024-11-07T14:13:58,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/209448dd382b4dbaba27d18c6e8010c2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/209448dd382b4dbaba27d18c6e8010c2 2024-11-07T14:13:58,152 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/209448dd382b4dbaba27d18c6e8010c2, entries=150, sequenceid=87, filesize=11.7 K 2024-11-07T14:13:58,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/df5f05275692478ab7ddedb50d18f2c8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/df5f05275692478ab7ddedb50d18f2c8 2024-11-07T14:13:58,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,163 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/df5f05275692478ab7ddedb50d18f2c8, entries=150, sequenceid=87, filesize=11.7 K 2024-11-07T14:13:58,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988898163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988898163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988898163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988898163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/8bd3810980d94a139a0253f860934967 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8bd3810980d94a139a0253f860934967 2024-11-07T14:13:58,178 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8bd3810980d94a139a0253f860934967, entries=150, sequenceid=87, filesize=11.7 K 2024-11-07T14:13:58,180 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 8133e138f01b5d4b631e25ead6cfe8cd in 984ms, sequenceid=87, compaction requested=false 2024-11-07T14:13:58,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:58,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:58,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-07T14:13:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-07T14:13:58,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T14:13:58,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1420 sec 2024-11-07T14:13:58,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1550 sec 2024-11-07T14:13:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T14:13:58,193 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:13:58,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:13:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-07T14:13:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T14:13:58,197 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:13:58,198 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:13:58,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:13:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T14:13:58,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T14:13:58,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:58,354 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T14:13:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/edbbd0a521b04bc9bd6f3b7336aeaae2 is 50, key is test_row_0/A:col10/1730988837522/Put/seqid=0 2024-11-07T14:13:58,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741855_1031 (size=12001) 2024-11-07T14:13:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T14:13:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:58,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:58,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988898576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988898672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988898672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988898672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988898683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988898692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:58,772 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/edbbd0a521b04bc9bd6f3b7336aeaae2 2024-11-07T14:13:58,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/1178f28d31944b618af71560d525ad43 is 50, key is test_row_0/B:col10/1730988837522/Put/seqid=0 2024-11-07T14:13:58,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741856_1032 (size=12001) 2024-11-07T14:13:58,812 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/1178f28d31944b618af71560d525ad43 2024-11-07T14:13:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T14:13:58,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/aa61c5995e9b4913939299a5832e22e6 is 50, key is test_row_0/C:col10/1730988837522/Put/seqid=0 2024-11-07T14:13:58,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741857_1033 (size=12001) 2024-11-07T14:13:58,841 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/aa61c5995e9b4913939299a5832e22e6 2024-11-07T14:13:58,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/edbbd0a521b04bc9bd6f3b7336aeaae2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/edbbd0a521b04bc9bd6f3b7336aeaae2 2024-11-07T14:13:58,864 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/edbbd0a521b04bc9bd6f3b7336aeaae2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T14:13:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/1178f28d31944b618af71560d525ad43 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1178f28d31944b618af71560d525ad43 2024-11-07T14:13:58,877 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1178f28d31944b618af71560d525ad43, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T14:13:58,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/aa61c5995e9b4913939299a5832e22e6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/aa61c5995e9b4913939299a5832e22e6 2024-11-07T14:13:58,894 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/aa61c5995e9b4913939299a5832e22e6, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T14:13:58,899 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8133e138f01b5d4b631e25ead6cfe8cd in 545ms, sequenceid=116, compaction requested=true 2024-11-07T14:13:58,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:58,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:58,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-07T14:13:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-07T14:13:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:58,905 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:13:58,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:13:58,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-07T14:13:58,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 704 msec 2024-11-07T14:13:58,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 714 msec 2024-11-07T14:13:58,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0d9ead8bee404ea4b89c491be223c9e5 is 50, key is test_row_0/A:col10/1730988838903/Put/seqid=0 2024-11-07T14:13:58,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741858_1034 (size=12001) 2024-11-07T14:13:58,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0d9ead8bee404ea4b89c491be223c9e5 2024-11-07T14:13:58,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/4415334790a24839b56ebc6c58cfcbb1 is 50, key is test_row_0/B:col10/1730988838903/Put/seqid=0 2024-11-07T14:13:58,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741859_1035 (size=12001) 2024-11-07T14:13:58,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/4415334790a24839b56ebc6c58cfcbb1 2024-11-07T14:13:58,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c422385a2fe548fd9216600cde38421b is 50, key is test_row_0/C:col10/1730988838903/Put/seqid=0 2024-11-07T14:13:59,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741860_1036 (size=12001) 2024-11-07T14:13:59,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988899056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988899173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T14:13:59,333 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:13:59,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:13:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-07T14:13:59,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:13:59,340 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:13:59,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:13:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T14:13:59,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988899393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c422385a2fe548fd9216600cde38421b 2024-11-07T14:13:59,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0d9ead8bee404ea4b89c491be223c9e5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d9ead8bee404ea4b89c491be223c9e5 2024-11-07T14:13:59,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d9ead8bee404ea4b89c491be223c9e5, entries=150, sequenceid=127, filesize=11.7 K 2024-11-07T14:13:59,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/4415334790a24839b56ebc6c58cfcbb1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4415334790a24839b56ebc6c58cfcbb1 2024-11-07T14:13:59,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4415334790a24839b56ebc6c58cfcbb1, entries=150, sequenceid=127, filesize=11.7 K 2024-11-07T14:13:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c422385a2fe548fd9216600cde38421b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c422385a2fe548fd9216600cde38421b 2024-11-07T14:13:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T14:13:59,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c422385a2fe548fd9216600cde38421b, entries=150, sequenceid=127, filesize=11.7 K 2024-11-07T14:13:59,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8133e138f01b5d4b631e25ead6cfe8cd in 557ms, sequenceid=127, compaction requested=true 2024-11-07T14:13:59,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:59,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:13:59,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:13:59,463 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:13:59,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:13:59,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:13:59,463 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:13:59,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:13:59,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:13:59,466 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:13:59,466 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:13:59,467 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,467 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/81069e70ac2f4785b58ce5332c53717c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/df5f05275692478ab7ddedb50d18f2c8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1178f28d31944b618af71560d525ad43, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4415334790a24839b56ebc6c58cfcbb1] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=47.0 K 2024-11-07T14:13:59,467 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:13:59,468 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:13:59,468 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,468 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2e898e260aae43639ca11646552629f9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/209448dd382b4dbaba27d18c6e8010c2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/edbbd0a521b04bc9bd6f3b7336aeaae2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d9ead8bee404ea4b89c491be223c9e5] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=47.0 K 2024-11-07T14:13:59,468 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 81069e70ac2f4785b58ce5332c53717c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730988835152 2024-11-07T14:13:59,469 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e898e260aae43639ca11646552629f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730988835152 2024-11-07T14:13:59,469 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting df5f05275692478ab7ddedb50d18f2c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1730988836324 2024-11-07T14:13:59,470 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 209448dd382b4dbaba27d18c6e8010c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1730988836324 2024-11-07T14:13:59,470 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 1178f28d31944b618af71560d525ad43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730988837467 2024-11-07T14:13:59,470 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting edbbd0a521b04bc9bd6f3b7336aeaae2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730988837467 2024-11-07T14:13:59,471 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4415334790a24839b56ebc6c58cfcbb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1730988838543 2024-11-07T14:13:59,471 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d9ead8bee404ea4b89c491be223c9e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1730988838543 2024-11-07T14:13:59,491 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:13:59,492 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:13:59,493 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7fd2d437823c46329a8bc51a4117a85e is 50, key is test_row_0/A:col10/1730988838903/Put/seqid=0 2024-11-07T14:13:59,493 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/92bdfbd5f396414e888cce2d8da58e27 is 50, key is test_row_0/B:col10/1730988838903/Put/seqid=0 2024-11-07T14:13:59,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741861_1037 (size=12275) 2024-11-07T14:13:59,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T14:13:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,506 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:13:59,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741862_1038 (size=12275) 2024-11-07T14:13:59,514 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/92bdfbd5f396414e888cce2d8da58e27 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/92bdfbd5f396414e888cce2d8da58e27 2024-11-07T14:13:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/63d6fdeb0761486e8d044ae704d52673 is 50, key is test_row_0/A:col10/1730988839044/Put/seqid=0 2024-11-07T14:13:59,525 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7fd2d437823c46329a8bc51a4117a85e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fd2d437823c46329a8bc51a4117a85e 2024-11-07T14:13:59,528 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into 92bdfbd5f396414e888cce2d8da58e27(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:13:59,528 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:59,528 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=12, startTime=1730988839463; duration=0sec 2024-11-07T14:13:59,529 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:13:59,529 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:13:59,529 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:13:59,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741863_1039 (size=12151) 2024-11-07T14:13:59,534 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/63d6fdeb0761486e8d044ae704d52673 2024-11-07T14:13:59,536 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:13:59,536 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:13:59,536 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,537 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/5dfe4826a3dd4a619d9cf3f4f5ea667f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8bd3810980d94a139a0253f860934967, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/aa61c5995e9b4913939299a5832e22e6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c422385a2fe548fd9216600cde38421b] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=47.0 K 2024-11-07T14:13:59,537 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dfe4826a3dd4a619d9cf3f4f5ea667f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730988835152 2024-11-07T14:13:59,538 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd3810980d94a139a0253f860934967, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1730988836324 2024-11-07T14:13:59,539 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 7fd2d437823c46329a8bc51a4117a85e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:13:59,539 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:59,539 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=12, startTime=1730988839463; duration=0sec 2024-11-07T14:13:59,540 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:13:59,540 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:13:59,540 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting aa61c5995e9b4913939299a5832e22e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730988837467 2024-11-07T14:13:59,545 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c422385a2fe548fd9216600cde38421b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1730988838543 2024-11-07T14:13:59,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/f28921fed8614383b04283fe499f9bb5 is 50, key is test_row_0/B:col10/1730988839044/Put/seqid=0 2024-11-07T14:13:59,566 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:13:59,566 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/79e8b1addf054065bad1594852787559 is 50, key is test_row_0/C:col10/1730988838903/Put/seqid=0 2024-11-07T14:13:59,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741864_1040 (size=12151) 2024-11-07T14:13:59,573 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/f28921fed8614383b04283fe499f9bb5 2024-11-07T14:13:59,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741865_1041 (size=12275) 2024-11-07T14:13:59,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/f86066ba1a09469987ed9b30e106f02b is 50, key is test_row_0/C:col10/1730988839044/Put/seqid=0 2024-11-07T14:13:59,595 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/79e8b1addf054065bad1594852787559 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/79e8b1addf054065bad1594852787559 2024-11-07T14:13:59,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741866_1042 (size=12151) 2024-11-07T14:13:59,606 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into 79e8b1addf054065bad1594852787559(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:13:59,606 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:59,607 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=12, startTime=1730988839463; duration=0sec 2024-11-07T14:13:59,607 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:13:59,607 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:13:59,607 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/f86066ba1a09469987ed9b30e106f02b 2024-11-07T14:13:59,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/63d6fdeb0761486e8d044ae704d52673 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/63d6fdeb0761486e8d044ae704d52673 2024-11-07T14:13:59,626 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/63d6fdeb0761486e8d044ae704d52673, entries=150, sequenceid=152, filesize=11.9 K 2024-11-07T14:13:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/f28921fed8614383b04283fe499f9bb5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f28921fed8614383b04283fe499f9bb5 2024-11-07T14:13:59,636 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f28921fed8614383b04283fe499f9bb5, entries=150, sequenceid=152, filesize=11.9 K 2024-11-07T14:13:59,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/f86066ba1a09469987ed9b30e106f02b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f86066ba1a09469987ed9b30e106f02b 2024-11-07T14:13:59,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f86066ba1a09469987ed9b30e106f02b, entries=150, sequenceid=152, filesize=11.9 K 2024-11-07T14:13:59,651 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 8133e138f01b5d4b631e25ead6cfe8cd in 144ms, sequenceid=152, compaction requested=false 2024-11-07T14:13:59,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:13:59,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-07T14:13:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-07T14:13:59,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T14:13:59,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 312 msec 2024-11-07T14:13:59,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 322 msec 2024-11-07T14:13:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T14:13:59,662 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:13:59,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:13:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-07T14:13:59,665 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:13:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T14:13:59,666 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:13:59,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:13:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:13:59,715 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:13:59,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:13:59,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/83d4a714af53465e88c0cf90d2893b6d is 50, key is test_row_1/A:col10/1730988839707/Put/seqid=0 2024-11-07T14:13:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741867_1043 (size=19317) 2024-11-07T14:13:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T14:13:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988899776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988899778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988899780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988899782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988899786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T14:13:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:59,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988899892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988899892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988899893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988899893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:13:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988899902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:13:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:13:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:13:59,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T14:13:59,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:13:59,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:13:59,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:59,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:13:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T14:14:00,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988900112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988900112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988900113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988900113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988900113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/83d4a714af53465e88c0cf90d2893b6d 2024-11-07T14:14:00,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T14:14:00,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:00,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:00,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:00,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/64bd845e304d4800b327f62df9ab52f5 is 50, key is test_row_1/B:col10/1730988839707/Put/seqid=0 2024-11-07T14:14:00,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741868_1044 (size=9757) 2024-11-07T14:14:00,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/64bd845e304d4800b327f62df9ab52f5 2024-11-07T14:14:00,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/e5505ed5c39e49c8a93388db641760ab is 50, key is test_row_1/C:col10/1730988839707/Put/seqid=0 2024-11-07T14:14:00,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741869_1045 (size=9757) 2024-11-07T14:14:00,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/e5505ed5c39e49c8a93388db641760ab 2024-11-07T14:14:00,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/83d4a714af53465e88c0cf90d2893b6d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/83d4a714af53465e88c0cf90d2893b6d 2024-11-07T14:14:00,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/83d4a714af53465e88c0cf90d2893b6d, entries=300, sequenceid=166, filesize=18.9 K 2024-11-07T14:14:00,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/64bd845e304d4800b327f62df9ab52f5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/64bd845e304d4800b327f62df9ab52f5 2024-11-07T14:14:00,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/64bd845e304d4800b327f62df9ab52f5, entries=100, sequenceid=166, filesize=9.5 K 2024-11-07T14:14:00,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/e5505ed5c39e49c8a93388db641760ab as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e5505ed5c39e49c8a93388db641760ab 2024-11-07T14:14:00,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e5505ed5c39e49c8a93388db641760ab, entries=100, sequenceid=166, filesize=9.5 K 2024-11-07T14:14:00,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8133e138f01b5d4b631e25ead6cfe8cd in 560ms, sequenceid=166, compaction requested=true 2024-11-07T14:14:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:00,276 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:00,276 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:00,278 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43743 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:00,278 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:00,278 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,279 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fd2d437823c46329a8bc51a4117a85e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/63d6fdeb0761486e8d044ae704d52673, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/83d4a714af53465e88c0cf90d2893b6d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=42.7 K 2024-11-07T14:14:00,279 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:00,279 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:00,279 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,280 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/92bdfbd5f396414e888cce2d8da58e27, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f28921fed8614383b04283fe499f9bb5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/64bd845e304d4800b327f62df9ab52f5] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=33.4 K 2024-11-07T14:14:00,280 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fd2d437823c46329a8bc51a4117a85e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1730988838543 2024-11-07T14:14:00,280 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bdfbd5f396414e888cce2d8da58e27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1730988838543 2024-11-07T14:14:00,280 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d6fdeb0761486e8d044ae704d52673, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730988839036 2024-11-07T14:14:00,281 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f28921fed8614383b04283fe499f9bb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730988839036 2024-11-07T14:14:00,283 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83d4a714af53465e88c0cf90d2893b6d, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988839700 2024-11-07T14:14:00,283 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 64bd845e304d4800b327f62df9ab52f5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988839707 2024-11-07T14:14:00,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T14:14:00,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,289 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:14:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T14:14:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/d71f45ff45a54c468b589f7d09f5ebad is 50, key is test_row_0/A:col10/1730988839784/Put/seqid=0 2024-11-07T14:14:00,303 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#35 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:00,303 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/b5fd8c36a22c452ca13beecbbc4df497 is 50, key is test_row_0/B:col10/1730988839044/Put/seqid=0 2024-11-07T14:14:00,303 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#34 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:00,304 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0bbee4b8b8aa4946a0e15e3254b59123 is 50, key is test_row_0/A:col10/1730988839044/Put/seqid=0 2024-11-07T14:14:00,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741870_1046 (size=12151) 2024-11-07T14:14:00,325 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/d71f45ff45a54c468b589f7d09f5ebad 2024-11-07T14:14:00,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741871_1047 (size=12527) 2024-11-07T14:14:00,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741872_1048 (size=12527) 2024-11-07T14:14:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/48e48f44638646feb57b7b43cbb5ef10 is 50, key is test_row_0/B:col10/1730988839784/Put/seqid=0 2024-11-07T14:14:00,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741873_1049 (size=12151) 2024-11-07T14:14:00,370 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/48e48f44638646feb57b7b43cbb5ef10 2024-11-07T14:14:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6f0bd571e4e541dba05ee07329c5d53b is 50, key is test_row_0/C:col10/1730988839784/Put/seqid=0 2024-11-07T14:14:00,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741874_1050 (size=12151) 2024-11-07T14:14:00,391 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6f0bd571e4e541dba05ee07329c5d53b 2024-11-07T14:14:00,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/d71f45ff45a54c468b589f7d09f5ebad as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d71f45ff45a54c468b589f7d09f5ebad 2024-11-07T14:14:00,404 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d71f45ff45a54c468b589f7d09f5ebad, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T14:14:00,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/48e48f44638646feb57b7b43cbb5ef10 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/48e48f44638646feb57b7b43cbb5ef10 2024-11-07T14:14:00,420 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/48e48f44638646feb57b7b43cbb5ef10, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T14:14:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6f0bd571e4e541dba05ee07329c5d53b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6f0bd571e4e541dba05ee07329c5d53b 2024-11-07T14:14:00,428 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6f0bd571e4e541dba05ee07329c5d53b, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T14:14:00,429 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 8133e138f01b5d4b631e25ead6cfe8cd in 141ms, sequenceid=191, compaction requested=true 2024-11-07T14:14:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-07T14:14:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-07T14:14:00,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T14:14:00,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 768 msec 2024-11-07T14:14:00,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 775 msec 2024-11-07T14:14:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:00,455 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:00,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:00,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/e843b13328064481bfcd1200cff16b38 is 50, key is test_row_0/A:col10/1730988840441/Put/seqid=0 2024-11-07T14:14:00,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741875_1051 (size=21715) 2024-11-07T14:14:00,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988900532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988900543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988900544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988900547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988900547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988900652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988900662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988900662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988900663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988900672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,768 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/b5fd8c36a22c452ca13beecbbc4df497 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b5fd8c36a22c452ca13beecbbc4df497 2024-11-07T14:14:00,768 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0bbee4b8b8aa4946a0e15e3254b59123 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0bbee4b8b8aa4946a0e15e3254b59123 2024-11-07T14:14:00,779 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 0bbee4b8b8aa4946a0e15e3254b59123(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:00,779 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:00,779 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=13, startTime=1730988840276; duration=0sec 2024-11-07T14:14:00,780 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:00,780 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:00,780 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:14:00,782 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46334 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:14:00,783 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:14:00,783 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,783 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into b5fd8c36a22c452ca13beecbbc4df497(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:00,783 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/79e8b1addf054065bad1594852787559, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f86066ba1a09469987ed9b30e106f02b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e5505ed5c39e49c8a93388db641760ab, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6f0bd571e4e541dba05ee07329c5d53b] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=45.2 K 2024-11-07T14:14:00,783 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:00,783 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=13, startTime=1730988840276; duration=0sec 2024-11-07T14:14:00,784 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:00,784 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:00,785 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79e8b1addf054065bad1594852787559, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1730988838543 2024-11-07T14:14:00,785 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86066ba1a09469987ed9b30e106f02b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730988839036 2024-11-07T14:14:00,786 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5505ed5c39e49c8a93388db641760ab, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988839707 2024-11-07T14:14:00,787 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f0bd571e4e541dba05ee07329c5d53b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730988839775 2024-11-07T14:14:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T14:14:00,803 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:00,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-07T14:14:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T14:14:00,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:00,816 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:00,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:00,828 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#39 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:00,829 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/0f9d932c80fd413a99431d570e9777d4 is 50, key is test_row_0/C:col10/1730988839784/Put/seqid=0 2024-11-07T14:14:00,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741876_1052 (size=12561) 2024-11-07T14:14:00,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988900862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/e843b13328064481bfcd1200cff16b38 2024-11-07T14:14:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/dbfd3b6a66f44504950f758cd129a3e2 is 50, key is test_row_0/B:col10/1730988840441/Put/seqid=0 2024-11-07T14:14:00,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741877_1053 (size=12151) 2024-11-07T14:14:00,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988900882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/dbfd3b6a66f44504950f758cd129a3e2 2024-11-07T14:14:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988900883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988900883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988900892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:00,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c76b992c865b4ba995b1f8a5f2a5d488 is 50, key is test_row_0/C:col10/1730988840441/Put/seqid=0 2024-11-07T14:14:00,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741878_1054 (size=12151) 2024-11-07T14:14:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T14:14:00,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T14:14:00,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:00,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:00,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:00,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T14:14:01,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T14:14:01,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:01,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:01,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988901172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988901192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988901202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988901202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988901212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,244 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/0f9d932c80fd413a99431d570e9777d4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0f9d932c80fd413a99431d570e9777d4 2024-11-07T14:14:01,254 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into 0f9d932c80fd413a99431d570e9777d4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:01,255 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:01,255 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=12, startTime=1730988840276; duration=0sec 2024-11-07T14:14:01,255 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:01,255 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:01,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T14:14:01,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:01,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:01,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c76b992c865b4ba995b1f8a5f2a5d488 2024-11-07T14:14:01,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/e843b13328064481bfcd1200cff16b38 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/e843b13328064481bfcd1200cff16b38 2024-11-07T14:14:01,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/e843b13328064481bfcd1200cff16b38, entries=350, sequenceid=202, filesize=21.2 K 2024-11-07T14:14:01,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/dbfd3b6a66f44504950f758cd129a3e2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/dbfd3b6a66f44504950f758cd129a3e2 2024-11-07T14:14:01,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/dbfd3b6a66f44504950f758cd129a3e2, entries=150, sequenceid=202, filesize=11.9 K 2024-11-07T14:14:01,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c76b992c865b4ba995b1f8a5f2a5d488 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c76b992c865b4ba995b1f8a5f2a5d488 2024-11-07T14:14:01,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c76b992c865b4ba995b1f8a5f2a5d488, entries=150, sequenceid=202, filesize=11.9 K 2024-11-07T14:14:01,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8133e138f01b5d4b631e25ead6cfe8cd in 914ms, sequenceid=202, compaction requested=true 2024-11-07T14:14:01,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:01,369 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:01,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:01,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:01,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:01,370 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:01,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:01,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:01,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:01,371 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 46393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:01,371 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:01,371 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:01,371 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0bbee4b8b8aa4946a0e15e3254b59123, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d71f45ff45a54c468b589f7d09f5ebad, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/e843b13328064481bfcd1200cff16b38] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=45.3 K 2024-11-07T14:14:01,372 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:01,372 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:01,372 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:01,372 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b5fd8c36a22c452ca13beecbbc4df497, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/48e48f44638646feb57b7b43cbb5ef10, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/dbfd3b6a66f44504950f758cd129a3e2] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=36.0 K 2024-11-07T14:14:01,373 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bbee4b8b8aa4946a0e15e3254b59123, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988839044 2024-11-07T14:14:01,373 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b5fd8c36a22c452ca13beecbbc4df497, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988839044 2024-11-07T14:14:01,373 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 48e48f44638646feb57b7b43cbb5ef10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730988839775 2024-11-07T14:14:01,373 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d71f45ff45a54c468b589f7d09f5ebad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730988839775 2024-11-07T14:14:01,374 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting dbfd3b6a66f44504950f758cd129a3e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730988840441 2024-11-07T14:14:01,374 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e843b13328064481bfcd1200cff16b38, keycount=350, bloomtype=ROW, size=21.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730988840437 2024-11-07T14:14:01,387 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:01,388 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/5ac126b1a8b747929fa9e61055c31bf2 is 50, key is test_row_0/A:col10/1730988840441/Put/seqid=0 2024-11-07T14:14:01,389 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:01,390 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/1669080b00d944aaa76aa878cda12a6a is 50, key is test_row_0/B:col10/1730988840441/Put/seqid=0 2024-11-07T14:14:01,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741879_1055 (size=12629) 2024-11-07T14:14:01,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741880_1056 (size=12629) 2024-11-07T14:14:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T14:14:01,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T14:14:01,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:01,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:14:01,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/dee6c413dd144fa89763c0542dd38bb5 is 50, key is test_row_0/A:col10/1730988840543/Put/seqid=0 2024-11-07T14:14:01,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741881_1057 (size=12151) 2024-11-07T14:14:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:01,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:01,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988901708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988901711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988901712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988901712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988901723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,817 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/5ac126b1a8b747929fa9e61055c31bf2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/5ac126b1a8b747929fa9e61055c31bf2 2024-11-07T14:14:01,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988901823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988901823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:01,830 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/1669080b00d944aaa76aa878cda12a6a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1669080b00d944aaa76aa878cda12a6a 2024-11-07T14:14:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:01,844 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 5ac126b1a8b747929fa9e61055c31bf2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:01,844 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:01,844 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=13, startTime=1730988841369; duration=0sec 2024-11-07T14:14:01,846 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into 1669080b00d944aaa76aa878cda12a6a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:01,846 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:01,846 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:01,846 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:01,846 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=13, startTime=1730988841370; duration=0sec 2024-11-07T14:14:01,846 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T14:14:01,846 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:01,846 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:01,850 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T14:14:01,851 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T14:14:01,851 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. because compaction request was cancelled 2024-11-07T14:14:01,851 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:01,934 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/dee6c413dd144fa89763c0542dd38bb5 2024-11-07T14:14:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T14:14:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6d5e04d3cd73453da11bdf16170ddbd5 is 50, key is test_row_0/B:col10/1730988840543/Put/seqid=0 2024-11-07T14:14:01,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741882_1058 (size=12151) 2024-11-07T14:14:01,989 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6d5e04d3cd73453da11bdf16170ddbd5 2024-11-07T14:14:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/2f888dc8e5ee42798328f0abeeec800c is 50, key is test_row_0/C:col10/1730988840543/Put/seqid=0 2024-11-07T14:14:02,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741883_1059 (size=12151) 2024-11-07T14:14:02,013 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/2f888dc8e5ee42798328f0abeeec800c 2024-11-07T14:14:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/dee6c413dd144fa89763c0542dd38bb5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dee6c413dd144fa89763c0542dd38bb5 2024-11-07T14:14:02,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988902033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,036 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dee6c413dd144fa89763c0542dd38bb5, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T14:14:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6d5e04d3cd73453da11bdf16170ddbd5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6d5e04d3cd73453da11bdf16170ddbd5 2024-11-07T14:14:02,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,046 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6d5e04d3cd73453da11bdf16170ddbd5, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T14:14:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988902043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/2f888dc8e5ee42798328f0abeeec800c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/2f888dc8e5ee42798328f0abeeec800c 2024-11-07T14:14:02,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/2f888dc8e5ee42798328f0abeeec800c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T14:14:02,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8133e138f01b5d4b631e25ead6cfe8cd in 610ms, sequenceid=230, compaction requested=true 2024-11-07T14:14:02,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:02,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:02,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-07T14:14:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-07T14:14:02,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-07T14:14:02,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2480 sec 2024-11-07T14:14:02,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2670 sec 2024-11-07T14:14:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:02,347 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:02,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:14:02,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0d81ab1bf49244c490f31609b5883788 is 50, key is test_row_0/A:col10/1730988841708/Put/seqid=0 2024-11-07T14:14:02,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741884_1060 (size=14541) 2024-11-07T14:14:02,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0d81ab1bf49244c490f31609b5883788 2024-11-07T14:14:02,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/e4c404e23eb54d7682f34936c54d8da3 is 50, key is test_row_0/B:col10/1730988841708/Put/seqid=0 2024-11-07T14:14:02,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741885_1061 (size=12151) 2024-11-07T14:14:02,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988902431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988902432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988902544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988902552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988902722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,728 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:02,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988902732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,736 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988902743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,750 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988902762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988902762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:02,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/e4c404e23eb54d7682f34936c54d8da3 2024-11-07T14:14:02,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d2294c9c5032436fb562c5bf85df2e0d is 50, key is test_row_0/C:col10/1730988841708/Put/seqid=0 2024-11-07T14:14:02,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741886_1062 (size=12151) 2024-11-07T14:14:02,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d2294c9c5032436fb562c5bf85df2e0d 2024-11-07T14:14:02,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0d81ab1bf49244c490f31609b5883788 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d81ab1bf49244c490f31609b5883788 2024-11-07T14:14:02,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d81ab1bf49244c490f31609b5883788, entries=200, sequenceid=244, filesize=14.2 K 2024-11-07T14:14:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T14:14:02,953 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:02,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/e4c404e23eb54d7682f34936c54d8da3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e4c404e23eb54d7682f34936c54d8da3 2024-11-07T14:14:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-07T14:14:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T14:14:02,959 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:02,960 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:02,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:02,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e4c404e23eb54d7682f34936c54d8da3, entries=150, sequenceid=244, filesize=11.9 K 2024-11-07T14:14:02,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d2294c9c5032436fb562c5bf85df2e0d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d2294c9c5032436fb562c5bf85df2e0d 2024-11-07T14:14:02,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d2294c9c5032436fb562c5bf85df2e0d, entries=150, sequenceid=244, filesize=11.9 K 2024-11-07T14:14:02,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8133e138f01b5d4b631e25ead6cfe8cd in 639ms, sequenceid=244, compaction requested=true 2024-11-07T14:14:02,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:02,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:02,988 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:02,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:02,988 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:02,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:02,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:02,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:02,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:02,998 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:02,998 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:02,998 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:02,998 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:02,999 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:02,999 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:02,999 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/5ac126b1a8b747929fa9e61055c31bf2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dee6c413dd144fa89763c0542dd38bb5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d81ab1bf49244c490f31609b5883788] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=38.4 K 2024-11-07T14:14:02,999 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1669080b00d944aaa76aa878cda12a6a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6d5e04d3cd73453da11bdf16170ddbd5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e4c404e23eb54d7682f34936c54d8da3] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=36.1 K 2024-11-07T14:14:03,000 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac126b1a8b747929fa9e61055c31bf2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730988840441 2024-11-07T14:14:03,000 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 1669080b00d944aaa76aa878cda12a6a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730988840441 2024-11-07T14:14:03,001 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting dee6c413dd144fa89763c0542dd38bb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730988840540 2024-11-07T14:14:03,001 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d5e04d3cd73453da11bdf16170ddbd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730988840540 2024-11-07T14:14:03,002 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d81ab1bf49244c490f31609b5883788, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730988841701 2024-11-07T14:14:03,003 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c404e23eb54d7682f34936c54d8da3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730988841701 2024-11-07T14:14:03,026 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:03,027 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/38bf5129872b4b3284cab0bcb941654d is 50, key is test_row_0/B:col10/1730988841708/Put/seqid=0 2024-11-07T14:14:03,031 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:03,032 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/39da152643a0400a870d152688bb809f is 50, key is test_row_0/A:col10/1730988841708/Put/seqid=0 2024-11-07T14:14:03,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741887_1063 (size=12731) 2024-11-07T14:14:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T14:14:03,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741888_1064 (size=12731) 2024-11-07T14:14:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:03,083 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:03,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T14:14:03,092 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/38bf5129872b4b3284cab0bcb941654d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/38bf5129872b4b3284cab0bcb941654d 2024-11-07T14:14:03,108 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/39da152643a0400a870d152688bb809f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/39da152643a0400a870d152688bb809f 2024-11-07T14:14:03,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/da82efbeefe64625b1a4851ed3ef5321 is 50, key is test_row_0/A:col10/1730988843077/Put/seqid=0 2024-11-07T14:14:03,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:03,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:03,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,131 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into 38bf5129872b4b3284cab0bcb941654d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:03,131 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:03,131 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=13, startTime=1730988842988; duration=0sec 2024-11-07T14:14:03,132 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:03,132 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:03,132 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:14:03,132 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 39da152643a0400a870d152688bb809f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:03,132 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:03,132 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=13, startTime=1730988842988; duration=0sec 2024-11-07T14:14:03,133 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:03,133 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:03,138 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:14:03,138 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:14:03,138 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,138 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0f9d932c80fd413a99431d570e9777d4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c76b992c865b4ba995b1f8a5f2a5d488, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/2f888dc8e5ee42798328f0abeeec800c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d2294c9c5032436fb562c5bf85df2e0d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=47.9 K 2024-11-07T14:14:03,140 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f9d932c80fd413a99431d570e9777d4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730988839775 2024-11-07T14:14:03,142 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c76b992c865b4ba995b1f8a5f2a5d488, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730988840441 2024-11-07T14:14:03,142 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f888dc8e5ee42798328f0abeeec800c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730988840540 2024-11-07T14:14:03,143 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d2294c9c5032436fb562c5bf85df2e0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730988841701 2024-11-07T14:14:03,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988903137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988903142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:03,170 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:03,171 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/fb94301e89d34ffaa42bccccb62b3c9b is 50, key is test_row_0/C:col10/1730988841708/Put/seqid=0 2024-11-07T14:14:03,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741889_1065 (size=12301) 2024-11-07T14:14:03,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741890_1066 (size=12697) 2024-11-07T14:14:03,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988903255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T14:14:03,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988903276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:03,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:03,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988903473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:03,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988903492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:03,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/da82efbeefe64625b1a4851ed3ef5321 2024-11-07T14:14:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T14:14:03,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/e46910b58abe49548ea158939c1dd552 is 50, key is test_row_0/B:col10/1730988843077/Put/seqid=0 2024-11-07T14:14:03,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:03,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,634 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/fb94301e89d34ffaa42bccccb62b3c9b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fb94301e89d34ffaa42bccccb62b3c9b 2024-11-07T14:14:03,643 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into fb94301e89d34ffaa42bccccb62b3c9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:03,643 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:03,643 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=12, startTime=1730988842989; duration=0sec 2024-11-07T14:14:03,643 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:03,643 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:03,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741891_1067 (size=12301) 2024-11-07T14:14:03,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/e46910b58abe49548ea158939c1dd552 2024-11-07T14:14:03,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/4de5d2bf86ac4227b4f15f3d7595051b is 50, key is test_row_0/C:col10/1730988843077/Put/seqid=0 2024-11-07T14:14:03,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741892_1068 (size=12301) 2024-11-07T14:14:03,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/4de5d2bf86ac4227b4f15f3d7595051b 2024-11-07T14:14:03,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/da82efbeefe64625b1a4851ed3ef5321 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/da82efbeefe64625b1a4851ed3ef5321 2024-11-07T14:14:03,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/da82efbeefe64625b1a4851ed3ef5321, entries=150, sequenceid=270, filesize=12.0 K 2024-11-07T14:14:03,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/e46910b58abe49548ea158939c1dd552 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e46910b58abe49548ea158939c1dd552 2024-11-07T14:14:03,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e46910b58abe49548ea158939c1dd552, entries=150, sequenceid=270, filesize=12.0 K 2024-11-07T14:14:03,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/4de5d2bf86ac4227b4f15f3d7595051b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4de5d2bf86ac4227b4f15f3d7595051b 2024-11-07T14:14:03,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4de5d2bf86ac4227b4f15f3d7595051b, entries=150, sequenceid=270, filesize=12.0 K 2024-11-07T14:14:03,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8133e138f01b5d4b631e25ead6cfe8cd in 706ms, sequenceid=270, compaction requested=false 2024-11-07T14:14:03,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:03,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:03,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:03,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7dc73c58eb3245839f270cc093033ffc is 50, key is test_row_0/A:col10/1730988843802/Put/seqid=0 2024-11-07T14:14:03,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741893_1069 (size=14741) 2024-11-07T14:14:03,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7dc73c58eb3245839f270cc093033ffc 2024-11-07T14:14:03,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/5f6e143da63b4c74b3fab6be8f18f9b9 is 50, key is test_row_0/B:col10/1730988843802/Put/seqid=0 2024-11-07T14:14:03,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:03,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:03,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:03,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:03,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988903935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:03,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988903941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:03,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741894_1070 (size=12301) 2024-11-07T14:14:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988904053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988904055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T14:14:04,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988904262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988904282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/5f6e143da63b4c74b3fab6be8f18f9b9 2024-11-07T14:14:04,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6cc96d7604ec4bc5b17742c53caed9c0 is 50, key is test_row_0/C:col10/1730988843802/Put/seqid=0 2024-11-07T14:14:04,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741895_1071 (size=12301) 2024-11-07T14:14:04,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:04,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:04,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988904572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988904593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:04,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:04,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988904743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,751 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988904763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,766 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:04,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6cc96d7604ec4bc5b17742c53caed9c0 2024-11-07T14:14:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988904772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:04,782 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7dc73c58eb3245839f270cc093033ffc as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7dc73c58eb3245839f270cc093033ffc 2024-11-07T14:14:04,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7dc73c58eb3245839f270cc093033ffc, entries=200, sequenceid=285, filesize=14.4 K 2024-11-07T14:14:04,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/5f6e143da63b4c74b3fab6be8f18f9b9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/5f6e143da63b4c74b3fab6be8f18f9b9 2024-11-07T14:14:04,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/5f6e143da63b4c74b3fab6be8f18f9b9, entries=150, sequenceid=285, filesize=12.0 K 2024-11-07T14:14:04,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/6cc96d7604ec4bc5b17742c53caed9c0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6cc96d7604ec4bc5b17742c53caed9c0 2024-11-07T14:14:04,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6cc96d7604ec4bc5b17742c53caed9c0, entries=150, sequenceid=285, filesize=12.0 K 2024-11-07T14:14:04,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8133e138f01b5d4b631e25ead6cfe8cd in 1048ms, sequenceid=285, compaction requested=true 2024-11-07T14:14:04,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:04,854 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:04,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T14:14:04,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:04,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:04,855 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:14:04,855 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:04,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:04,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:04,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:04,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:04,857 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:04,857 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:04,858 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,858 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/39da152643a0400a870d152688bb809f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/da82efbeefe64625b1a4851ed3ef5321, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7dc73c58eb3245839f270cc093033ffc] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=38.8 K 2024-11-07T14:14:04,858 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:04,858 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:04,858 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,858 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/38bf5129872b4b3284cab0bcb941654d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e46910b58abe49548ea158939c1dd552, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/5f6e143da63b4c74b3fab6be8f18f9b9] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=36.5 K 2024-11-07T14:14:04,861 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 38bf5129872b4b3284cab0bcb941654d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730988841701 2024-11-07T14:14:04,862 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e46910b58abe49548ea158939c1dd552, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988842389 2024-11-07T14:14:04,863 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6e143da63b4c74b3fab6be8f18f9b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1730988843136 2024-11-07T14:14:04,864 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39da152643a0400a870d152688bb809f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730988841701 2024-11-07T14:14:04,865 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting da82efbeefe64625b1a4851ed3ef5321, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988842389 2024-11-07T14:14:04,867 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dc73c58eb3245839f270cc093033ffc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1730988843123 2024-11-07T14:14:04,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7278585dd1f04204a098753932209e91 is 50, key is test_row_0/A:col10/1730988843921/Put/seqid=0 2024-11-07T14:14:04,884 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:04,885 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/7956ac91fe1341bd9fee0e2327465f27 is 50, key is test_row_0/B:col10/1730988843802/Put/seqid=0 2024-11-07T14:14:04,906 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:04,907 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/3b56940063ad440c8a745ef262913221 is 50, key is test_row_0/A:col10/1730988843802/Put/seqid=0 2024-11-07T14:14:04,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741896_1072 (size=12301) 2024-11-07T14:14:04,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741897_1073 (size=12983) 2024-11-07T14:14:04,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741898_1074 (size=12983) 2024-11-07T14:14:04,960 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/7956ac91fe1341bd9fee0e2327465f27 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7956ac91fe1341bd9fee0e2327465f27 2024-11-07T14:14:04,965 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/3b56940063ad440c8a745ef262913221 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/3b56940063ad440c8a745ef262913221 2024-11-07T14:14:04,978 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into 7956ac91fe1341bd9fee0e2327465f27(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:04,978 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:04,978 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=13, startTime=1730988844855; duration=0sec 2024-11-07T14:14:04,979 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:04,979 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:04,979 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:04,983 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 3b56940063ad440c8a745ef262913221(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:04,984 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:04,984 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=13, startTime=1730988844853; duration=0sec 2024-11-07T14:14:04,984 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:04,984 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:14:04,984 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:04,985 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fb94301e89d34ffaa42bccccb62b3c9b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4de5d2bf86ac4227b4f15f3d7595051b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6cc96d7604ec4bc5b17742c53caed9c0] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=36.4 K 2024-11-07T14:14:04,985 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:04,985 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:04,986 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting fb94301e89d34ffaa42bccccb62b3c9b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1730988841701 2024-11-07T14:14:04,987 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de5d2bf86ac4227b4f15f3d7595051b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988842389 2024-11-07T14:14:04,987 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc96d7604ec4bc5b17742c53caed9c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1730988843136 2024-11-07T14:14:05,041 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:05,042 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c81a20c1a3e84fb89051f2e12eedb1bd is 50, key is test_row_0/C:col10/1730988843802/Put/seqid=0 2024-11-07T14:14:05,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:05,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741899_1075 (size=12949) 2024-11-07T14:14:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T14:14:05,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988905118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:05,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988905119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:05,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988905233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:05,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988905233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:05,326 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7278585dd1f04204a098753932209e91 2024-11-07T14:14:05,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6dd6217babfd4000b012c6abda09e6d3 is 50, key is test_row_0/B:col10/1730988843921/Put/seqid=0 2024-11-07T14:14:05,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741900_1076 (size=12301) 2024-11-07T14:14:05,349 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6dd6217babfd4000b012c6abda09e6d3 2024-11-07T14:14:05,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d917b6d7409b4df499ceab5ad9a3d304 is 50, key is test_row_0/C:col10/1730988843921/Put/seqid=0 2024-11-07T14:14:05,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741901_1077 (size=12301) 2024-11-07T14:14:05,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988905442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988905442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:05,499 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c81a20c1a3e84fb89051f2e12eedb1bd as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c81a20c1a3e84fb89051f2e12eedb1bd 2024-11-07T14:14:05,509 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into c81a20c1a3e84fb89051f2e12eedb1bd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:05,509 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:05,509 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=13, startTime=1730988844856; duration=0sec 2024-11-07T14:14:05,510 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:05,510 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:05,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988905753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:05,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988905753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:05,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:05,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:05,766 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d917b6d7409b4df499ceab5ad9a3d304 2024-11-07T14:14:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/7278585dd1f04204a098753932209e91 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7278585dd1f04204a098753932209e91 2024-11-07T14:14:05,797 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7278585dd1f04204a098753932209e91, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T14:14:05,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/6dd6217babfd4000b012c6abda09e6d3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6dd6217babfd4000b012c6abda09e6d3 2024-11-07T14:14:05,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6dd6217babfd4000b012c6abda09e6d3, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T14:14:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d917b6d7409b4df499ceab5ad9a3d304 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d917b6d7409b4df499ceab5ad9a3d304 2024-11-07T14:14:05,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d917b6d7409b4df499ceab5ad9a3d304, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T14:14:05,833 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8133e138f01b5d4b631e25ead6cfe8cd in 978ms, sequenceid=309, compaction requested=false 2024-11-07T14:14:05,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:05,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:05,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-07T14:14:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-07T14:14:05,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T14:14:05,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8770 sec 2024-11-07T14:14:05,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.8900 sec 2024-11-07T14:14:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:06,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T14:14:06,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0fe83a007d3d40d79eeabf6a47e2c25d is 50, key is test_row_0/A:col10/1730988846266/Put/seqid=0 2024-11-07T14:14:06,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741902_1078 (size=12301) 2024-11-07T14:14:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988906351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988906351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988906462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988906462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:06,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988906672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:06,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988906683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:06,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0fe83a007d3d40d79eeabf6a47e2c25d 2024-11-07T14:14:06,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/2d236fac82af470282360f6c20d4277f is 50, key is test_row_0/B:col10/1730988846266/Put/seqid=0 2024-11-07T14:14:06,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741903_1079 (size=12301) 2024-11-07T14:14:06,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/2d236fac82af470282360f6c20d4277f 2024-11-07T14:14:06,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/46d4e38fc20b44d3845d0589a784e28a is 50, key is test_row_0/C:col10/1730988846266/Put/seqid=0 2024-11-07T14:14:06,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741904_1080 (size=12301) 2024-11-07T14:14:06,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/46d4e38fc20b44d3845d0589a784e28a 2024-11-07T14:14:06,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/0fe83a007d3d40d79eeabf6a47e2c25d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0fe83a007d3d40d79eeabf6a47e2c25d 2024-11-07T14:14:06,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0fe83a007d3d40d79eeabf6a47e2c25d, entries=150, sequenceid=326, filesize=12.0 K 2024-11-07T14:14:06,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/2d236fac82af470282360f6c20d4277f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2d236fac82af470282360f6c20d4277f 2024-11-07T14:14:06,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2d236fac82af470282360f6c20d4277f, entries=150, sequenceid=326, filesize=12.0 K 2024-11-07T14:14:06,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/46d4e38fc20b44d3845d0589a784e28a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/46d4e38fc20b44d3845d0589a784e28a 2024-11-07T14:14:06,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/46d4e38fc20b44d3845d0589a784e28a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-07T14:14:06,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 8133e138f01b5d4b631e25ead6cfe8cd in 572ms, sequenceid=326, compaction requested=true 2024-11-07T14:14:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:06,840 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:06,841 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:06,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:06,843 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:06,843 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:06,844 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:06,844 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:06,844 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:06,844 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:06,844 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7956ac91fe1341bd9fee0e2327465f27, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6dd6217babfd4000b012c6abda09e6d3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2d236fac82af470282360f6c20d4277f] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=36.7 K 2024-11-07T14:14:06,844 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/3b56940063ad440c8a745ef262913221, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7278585dd1f04204a098753932209e91, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0fe83a007d3d40d79eeabf6a47e2c25d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=36.7 K 2024-11-07T14:14:06,845 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 7956ac91fe1341bd9fee0e2327465f27, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1730988843136 2024-11-07T14:14:06,846 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b56940063ad440c8a745ef262913221, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1730988843136 2024-11-07T14:14:06,847 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dd6217babfd4000b012c6abda09e6d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988843921 2024-11-07T14:14:06,850 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d236fac82af470282360f6c20d4277f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1730988845109 2024-11-07T14:14:06,850 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7278585dd1f04204a098753932209e91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988843921 2024-11-07T14:14:06,851 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe83a007d3d40d79eeabf6a47e2c25d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1730988845109 2024-11-07T14:14:06,885 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:06,886 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/44299a082c7c420aa5e4c2839cb31e52 is 50, key is test_row_0/B:col10/1730988846266/Put/seqid=0 2024-11-07T14:14:06,890 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:06,891 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/863135e0231a4882afc5f5bb59b2b060 is 50, key is test_row_0/A:col10/1730988846266/Put/seqid=0 2024-11-07T14:14:06,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741905_1081 (size=13085) 2024-11-07T14:14:06,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741906_1082 (size=13085) 2024-11-07T14:14:06,921 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/863135e0231a4882afc5f5bb59b2b060 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/863135e0231a4882afc5f5bb59b2b060 2024-11-07T14:14:06,931 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 863135e0231a4882afc5f5bb59b2b060(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:06,931 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:06,931 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=13, startTime=1730988846840; duration=0sec 2024-11-07T14:14:06,931 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:06,931 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:06,931 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:06,935 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:06,935 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:14:06,935 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:06,935 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c81a20c1a3e84fb89051f2e12eedb1bd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d917b6d7409b4df499ceab5ad9a3d304, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/46d4e38fc20b44d3845d0589a784e28a] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=36.7 K 2024-11-07T14:14:06,936 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c81a20c1a3e84fb89051f2e12eedb1bd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1730988843136 2024-11-07T14:14:06,936 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d917b6d7409b4df499ceab5ad9a3d304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988843921 2024-11-07T14:14:06,937 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d4e38fc20b44d3845d0589a784e28a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1730988845109 2024-11-07T14:14:06,956 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:06,957 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/fd97fc0a584446e29efbed07efef2735 is 50, key is test_row_0/C:col10/1730988846266/Put/seqid=0 2024-11-07T14:14:06,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741907_1083 (size=13051) 2024-11-07T14:14:06,974 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/fd97fc0a584446e29efbed07efef2735 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fd97fc0a584446e29efbed07efef2735 2024-11-07T14:14:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:06,985 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:06,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:14:06,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/19faa427a471489f9c53a67eb1573148 is 50, key is test_row_0/A:col10/1730988846339/Put/seqid=0 2024-11-07T14:14:06,997 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into fd97fc0a584446e29efbed07efef2735(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:06,997 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:06,997 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=13, startTime=1730988846841; duration=0sec 2024-11-07T14:14:06,997 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:06,997 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:07,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988907019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988907019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741908_1084 (size=14741) 2024-11-07T14:14:07,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/19faa427a471489f9c53a67eb1573148 2024-11-07T14:14:07,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/c287934e5b854be2b0be257fa93a3e77 is 50, key is test_row_0/B:col10/1730988846339/Put/seqid=0 2024-11-07T14:14:07,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741909_1085 (size=12301) 2024-11-07T14:14:07,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/c287934e5b854be2b0be257fa93a3e77 2024-11-07T14:14:07,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/0b4f981dcc40427bb995201826033163 is 50, key is test_row_0/C:col10/1730988846339/Put/seqid=0 2024-11-07T14:14:07,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741910_1086 (size=12301) 2024-11-07T14:14:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T14:14:07,123 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:07,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-07T14:14:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T14:14:07,128 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:07,129 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:07,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:07,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988907133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988907133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T14:14:07,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T14:14:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:07,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,324 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/44299a082c7c420aa5e4c2839cb31e52 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/44299a082c7c420aa5e4c2839cb31e52 2024-11-07T14:14:07,336 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into 44299a082c7c420aa5e4c2839cb31e52(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:07,336 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:07,336 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=13, startTime=1730988846841; duration=0sec 2024-11-07T14:14:07,337 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:07,337 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:07,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988907342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988907342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T14:14:07,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:07,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T14:14:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/0b4f981dcc40427bb995201826033163 2024-11-07T14:14:07,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/19faa427a471489f9c53a67eb1573148 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/19faa427a471489f9c53a67eb1573148 2024-11-07T14:14:07,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/19faa427a471489f9c53a67eb1573148, entries=200, sequenceid=351, filesize=14.4 K 2024-11-07T14:14:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/c287934e5b854be2b0be257fa93a3e77 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/c287934e5b854be2b0be257fa93a3e77 2024-11-07T14:14:07,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/c287934e5b854be2b0be257fa93a3e77, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T14:14:07,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/0b4f981dcc40427bb995201826033163 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0b4f981dcc40427bb995201826033163 2024-11-07T14:14:07,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0b4f981dcc40427bb995201826033163, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T14:14:07,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8133e138f01b5d4b631e25ead6cfe8cd in 563ms, sequenceid=351, compaction requested=false 2024-11-07T14:14:07,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:07,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T14:14:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,595 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:14:07,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/90892e474fa9449bafd68419dc2a7723 is 50, key is test_row_0/A:col10/1730988847016/Put/seqid=0 2024-11-07T14:14:07,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741911_1087 (size=12301) 2024-11-07T14:14:07,616 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/90892e474fa9449bafd68419dc2a7723 2024-11-07T14:14:07,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/ba523c2240194b9486205359d1c01b9a is 50, key is test_row_0/B:col10/1730988847016/Put/seqid=0 2024-11-07T14:14:07,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741912_1088 (size=12301) 2024-11-07T14:14:07,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/ba523c2240194b9486205359d1c01b9a 2024-11-07T14:14:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/b669da45da104ba3a40e6fca320a6973 is 50, key is test_row_0/C:col10/1730988847016/Put/seqid=0 2024-11-07T14:14:07,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741913_1089 (size=12301) 2024-11-07T14:14:07,677 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/b669da45da104ba3a40e6fca320a6973 2024-11-07T14:14:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:07,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/90892e474fa9449bafd68419dc2a7723 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/90892e474fa9449bafd68419dc2a7723 2024-11-07T14:14:07,692 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/90892e474fa9449bafd68419dc2a7723, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T14:14:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/ba523c2240194b9486205359d1c01b9a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/ba523c2240194b9486205359d1c01b9a 2024-11-07T14:14:07,701 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/ba523c2240194b9486205359d1c01b9a, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T14:14:07,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/b669da45da104ba3a40e6fca320a6973 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b669da45da104ba3a40e6fca320a6973 2024-11-07T14:14:07,710 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b669da45da104ba3a40e6fca320a6973, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T14:14:07,712 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=73.80 KB/75570 for 8133e138f01b5d4b631e25ead6cfe8cd in 116ms, sequenceid=365, compaction requested=true 2024-11-07T14:14:07,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:07,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-07T14:14:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-07T14:14:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:07,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:07,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T14:14:07,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T14:14:07,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 584 msec 2024-11-07T14:14:07,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 596 msec 2024-11-07T14:14:07,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/32826b16ac124888b3fe76af9a123d9d is 50, key is test_row_1/A:col10/1730988847706/Put/seqid=0 2024-11-07T14:14:07,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741914_1090 (size=12297) 2024-11-07T14:14:07,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/32826b16ac124888b3fe76af9a123d9d 2024-11-07T14:14:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T14:14:07,753 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:07,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-07T14:14:07,756 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T14:14:07,758 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:07,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:07,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/98ae7707f44340bcab663d5a1ca2101c is 50, key is test_row_1/B:col10/1730988847706/Put/seqid=0 2024-11-07T14:14:07,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988907765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988907767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741915_1091 (size=9857) 2024-11-07T14:14:07,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/98ae7707f44340bcab663d5a1ca2101c 2024-11-07T14:14:07,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c144ca4eee7c42afa9f147f67e9f8285 is 50, key is test_row_1/C:col10/1730988847706/Put/seqid=0 2024-11-07T14:14:07,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741916_1092 (size=9857) 2024-11-07T14:14:07,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c144ca4eee7c42afa9f147f67e9f8285 2024-11-07T14:14:07,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/32826b16ac124888b3fe76af9a123d9d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/32826b16ac124888b3fe76af9a123d9d 2024-11-07T14:14:07,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/32826b16ac124888b3fe76af9a123d9d, entries=150, sequenceid=382, filesize=12.0 K 2024-11-07T14:14:07,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/98ae7707f44340bcab663d5a1ca2101c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/98ae7707f44340bcab663d5a1ca2101c 2024-11-07T14:14:07,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/98ae7707f44340bcab663d5a1ca2101c, entries=100, sequenceid=382, filesize=9.6 K 2024-11-07T14:14:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c144ca4eee7c42afa9f147f67e9f8285 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c144ca4eee7c42afa9f147f67e9f8285 2024-11-07T14:14:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T14:14:07,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c144ca4eee7c42afa9f147f67e9f8285, entries=100, sequenceid=382, filesize=9.6 K 2024-11-07T14:14:07,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 8133e138f01b5d4b631e25ead6cfe8cd in 153ms, sequenceid=382, compaction requested=true 2024-11-07T14:14:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:07,867 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:14:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:07,867 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:14:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:07,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:07,870 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52424 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:14:07,870 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:07,870 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,871 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:14:07,871 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:07,871 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,871 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/44299a082c7c420aa5e4c2839cb31e52, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/c287934e5b854be2b0be257fa93a3e77, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/ba523c2240194b9486205359d1c01b9a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/98ae7707f44340bcab663d5a1ca2101c] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=46.4 K 2024-11-07T14:14:07,871 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/863135e0231a4882afc5f5bb59b2b060, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/19faa427a471489f9c53a67eb1573148, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/90892e474fa9449bafd68419dc2a7723, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/32826b16ac124888b3fe76af9a123d9d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=51.2 K 2024-11-07T14:14:07,871 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44299a082c7c420aa5e4c2839cb31e52, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1730988845109 2024-11-07T14:14:07,873 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 863135e0231a4882afc5f5bb59b2b060, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1730988845109 2024-11-07T14:14:07,874 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c287934e5b854be2b0be257fa93a3e77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730988846339 2024-11-07T14:14:07,874 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 19faa427a471489f9c53a67eb1573148, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730988846339 2024-11-07T14:14:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:07,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:07,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T14:14:07,875 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 90892e474fa9449bafd68419dc2a7723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730988847007 2024-11-07T14:14:07,876 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba523c2240194b9486205359d1c01b9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730988847007 2024-11-07T14:14:07,877 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 32826b16ac124888b3fe76af9a123d9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988847706 2024-11-07T14:14:07,879 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98ae7707f44340bcab663d5a1ca2101c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988847706 2024-11-07T14:14:07,901 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:07,902 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#80 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:07,902 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/f7ec1271cd6147639e026ac35fc0a9c8 is 50, key is test_row_0/B:col10/1730988847016/Put/seqid=0 2024-11-07T14:14:07,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/9ccc75344d914ea58e114feac03d32d1 is 50, key is test_row_0/A:col10/1730988847873/Put/seqid=0 2024-11-07T14:14:07,906 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/9be20dc3d2a14fa7941f8b89dda2000e is 50, key is test_row_0/A:col10/1730988847016/Put/seqid=0 2024-11-07T14:14:07,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T14:14:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:07,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741919_1095 (size=13221) 2024-11-07T14:14:07,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741917_1093 (size=14741) 2024-11-07T14:14:07,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/9ccc75344d914ea58e114feac03d32d1 2024-11-07T14:14:07,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988907921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:07,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988907924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:07,938 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/9be20dc3d2a14fa7941f8b89dda2000e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9be20dc3d2a14fa7941f8b89dda2000e 2024-11-07T14:14:07,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/0a6edc35144a479cb9720295edbdf19f is 50, key is test_row_0/B:col10/1730988847873/Put/seqid=0 2024-11-07T14:14:07,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741918_1094 (size=13221) 2024-11-07T14:14:07,950 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 9be20dc3d2a14fa7941f8b89dda2000e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:07,950 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:07,950 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=12, startTime=1730988847867; duration=0sec 2024-11-07T14:14:07,950 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:07,950 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:07,950 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:14:07,954 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:14:07,954 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:14:07,954 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:07,954 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fd97fc0a584446e29efbed07efef2735, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0b4f981dcc40427bb995201826033163, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b669da45da104ba3a40e6fca320a6973, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c144ca4eee7c42afa9f147f67e9f8285] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=46.4 K 2024-11-07T14:14:07,954 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting fd97fc0a584446e29efbed07efef2735, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1730988845109 2024-11-07T14:14:07,955 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b4f981dcc40427bb995201826033163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730988846339 2024-11-07T14:14:07,955 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b669da45da104ba3a40e6fca320a6973, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730988847007 2024-11-07T14:14:07,956 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c144ca4eee7c42afa9f147f67e9f8285, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988847706 2024-11-07T14:14:07,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741920_1096 (size=12301) 2024-11-07T14:14:07,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/0a6edc35144a479cb9720295edbdf19f 2024-11-07T14:14:07,968 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:07,969 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/3296a267dd7941738608bceb9b699ee1 is 50, key is test_row_0/C:col10/1730988847016/Put/seqid=0 2024-11-07T14:14:07,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741921_1097 (size=13187) 2024-11-07T14:14:07,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c6b683d784494cc49ab20f42c78578d4 is 50, key is test_row_0/C:col10/1730988847873/Put/seqid=0 2024-11-07T14:14:08,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741922_1098 (size=12301) 2024-11-07T14:14:08,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988908032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988908032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T14:14:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:08,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T14:14:08,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T14:14:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:08,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988908242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988908246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,363 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/f7ec1271cd6147639e026ac35fc0a9c8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f7ec1271cd6147639e026ac35fc0a9c8 2024-11-07T14:14:08,373 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into f7ec1271cd6147639e026ac35fc0a9c8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:08,373 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:08,373 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=12, startTime=1730988847867; duration=0sec 2024-11-07T14:14:08,374 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:08,374 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:08,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T14:14:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:08,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T14:14:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:08,396 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/3296a267dd7941738608bceb9b699ee1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/3296a267dd7941738608bceb9b699ee1 2024-11-07T14:14:08,403 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into 3296a267dd7941738608bceb9b699ee1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:08,403 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:08,403 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=12, startTime=1730988847867; duration=0sec 2024-11-07T14:14:08,404 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:08,404 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:08,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c6b683d784494cc49ab20f42c78578d4 2024-11-07T14:14:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/9ccc75344d914ea58e114feac03d32d1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9ccc75344d914ea58e114feac03d32d1 2024-11-07T14:14:08,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9ccc75344d914ea58e114feac03d32d1, entries=200, sequenceid=402, filesize=14.4 K 2024-11-07T14:14:08,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/0a6edc35144a479cb9720295edbdf19f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/0a6edc35144a479cb9720295edbdf19f 2024-11-07T14:14:08,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/0a6edc35144a479cb9720295edbdf19f, entries=150, sequenceid=402, filesize=12.0 K 2024-11-07T14:14:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c6b683d784494cc49ab20f42c78578d4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c6b683d784494cc49ab20f42c78578d4 2024-11-07T14:14:08,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c6b683d784494cc49ab20f42c78578d4, entries=150, sequenceid=402, filesize=12.0 K 2024-11-07T14:14:08,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 8133e138f01b5d4b631e25ead6cfe8cd in 560ms, sequenceid=402, compaction requested=false 2024-11-07T14:14:08,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:08,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T14:14:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:08,535 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T14:14:08,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/699e479f1dfe4d77bcdb2b1b242d6ce6 is 50, key is test_row_0/A:col10/1730988847921/Put/seqid=0 2024-11-07T14:14:08,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:08,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741923_1099 (size=12301) 2024-11-07T14:14:08,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988908607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988908609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988908722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988908722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40012 deadline: 1730988908772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,774 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40034 deadline: 1730988908812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,815 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40026 deadline: 1730988908832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,834 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 8133e138f01b5d4b631e25ead6cfe8cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T14:14:08,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988908933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988908933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:08,967 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/699e479f1dfe4d77bcdb2b1b242d6ce6 2024-11-07T14:14:08,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/4d1dd28812994d70a6893692fc611f55 is 50, key is test_row_0/B:col10/1730988847921/Put/seqid=0 2024-11-07T14:14:08,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741924_1100 (size=12301) 2024-11-07T14:14:09,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:09,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988909253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988909253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:09,397 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/4d1dd28812994d70a6893692fc611f55 2024-11-07T14:14:09,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/370e9a22af2f4e33a718d6088d6bb441 is 50, key is test_row_0/C:col10/1730988847921/Put/seqid=0 2024-11-07T14:14:09,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741925_1101 (size=12301) 2024-11-07T14:14:09,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988909762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988909772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:09,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/370e9a22af2f4e33a718d6088d6bb441 2024-11-07T14:14:09,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/699e479f1dfe4d77bcdb2b1b242d6ce6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/699e479f1dfe4d77bcdb2b1b242d6ce6 2024-11-07T14:14:09,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/699e479f1dfe4d77bcdb2b1b242d6ce6, entries=150, sequenceid=421, filesize=12.0 K 2024-11-07T14:14:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/4d1dd28812994d70a6893692fc611f55 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4d1dd28812994d70a6893692fc611f55 2024-11-07T14:14:09,839 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4d1dd28812994d70a6893692fc611f55, entries=150, sequenceid=421, filesize=12.0 K 2024-11-07T14:14:09,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/370e9a22af2f4e33a718d6088d6bb441 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/370e9a22af2f4e33a718d6088d6bb441 2024-11-07T14:14:09,850 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/370e9a22af2f4e33a718d6088d6bb441, entries=150, sequenceid=421, filesize=12.0 K 2024-11-07T14:14:09,852 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8133e138f01b5d4b631e25ead6cfe8cd in 1317ms, sequenceid=421, compaction requested=true 2024-11-07T14:14:09,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:09,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:09,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-07T14:14:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-07T14:14:09,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-07T14:14:09,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0960 sec 2024-11-07T14:14:09,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1040 sec 2024-11-07T14:14:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T14:14:09,903 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:09,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-07T14:14:09,906 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T14:14:09,908 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:09,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T14:14:10,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T14:14:10,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:10,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T14:14:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/2d76e9c0c10c42cfbf8912c23165830d is 50, key is test_row_0/A:col10/1730988848581/Put/seqid=0 2024-11-07T14:14:10,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741926_1102 (size=12301) 2024-11-07T14:14:10,090 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/2d76e9c0c10c42cfbf8912c23165830d 2024-11-07T14:14:10,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/d0258f31f6184b7b979be4146e97abc2 is 50, key is test_row_0/B:col10/1730988848581/Put/seqid=0 2024-11-07T14:14:10,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741927_1103 (size=12301) 2024-11-07T14:14:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T14:14:10,515 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/d0258f31f6184b7b979be4146e97abc2 2024-11-07T14:14:10,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/afaf19406e324030960613932e64b24e is 50, key is test_row_0/C:col10/1730988848581/Put/seqid=0 2024-11-07T14:14:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T14:14:10,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741928_1104 (size=12301) 2024-11-07T14:14:10,534 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/afaf19406e324030960613932e64b24e 2024-11-07T14:14:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/2d76e9c0c10c42cfbf8912c23165830d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2d76e9c0c10c42cfbf8912c23165830d 2024-11-07T14:14:10,546 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2d76e9c0c10c42cfbf8912c23165830d, entries=150, sequenceid=441, filesize=12.0 K 2024-11-07T14:14:10,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/d0258f31f6184b7b979be4146e97abc2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/d0258f31f6184b7b979be4146e97abc2 2024-11-07T14:14:10,553 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/d0258f31f6184b7b979be4146e97abc2, entries=150, sequenceid=441, filesize=12.0 K 2024-11-07T14:14:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/afaf19406e324030960613932e64b24e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/afaf19406e324030960613932e64b24e 2024-11-07T14:14:10,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/afaf19406e324030960613932e64b24e, entries=150, sequenceid=441, filesize=12.0 K 2024-11-07T14:14:10,562 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 8133e138f01b5d4b631e25ead6cfe8cd in 501ms, sequenceid=441, compaction requested=true 2024-11-07T14:14:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-07T14:14:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-07T14:14:10,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T14:14:10,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 656 msec 2024-11-07T14:14:10,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 668 msec 2024-11-07T14:14:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:10,798 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:10,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:10,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/fe1bafbd9df6437e8d3441bfb8ef15eb is 50, key is test_row_0/A:col10/1730988850791/Put/seqid=0 2024-11-07T14:14:10,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741929_1105 (size=14741) 2024-11-07T14:14:10,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988910851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:10,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988910856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988910962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988910962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T14:14:11,043 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:11,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-07T14:14:11,045 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T14:14:11,046 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:11,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T14:14:11,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988911172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988911172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:11,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T14:14:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/fe1bafbd9df6437e8d3441bfb8ef15eb 2024-11-07T14:14:11,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/14d5a92110394ece863ff042c4f8f073 is 50, key is test_row_0/B:col10/1730988850791/Put/seqid=0 2024-11-07T14:14:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741930_1106 (size=12301) 2024-11-07T14:14:11,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T14:14:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T14:14:11,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988911483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:11,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988911483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:11,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:11,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:11,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:11,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T14:14:11,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:11,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/14d5a92110394ece863ff042c4f8f073 2024-11-07T14:14:11,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d8464a5ac12f41dcbe64dd345bc16681 is 50, key is test_row_0/C:col10/1730988850791/Put/seqid=0 2024-11-07T14:14:11,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741931_1107 (size=12301) 2024-11-07T14:14:11,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d8464a5ac12f41dcbe64dd345bc16681 2024-11-07T14:14:11,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/fe1bafbd9df6437e8d3441bfb8ef15eb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/fe1bafbd9df6437e8d3441bfb8ef15eb 2024-11-07T14:14:11,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/fe1bafbd9df6437e8d3441bfb8ef15eb, entries=200, sequenceid=452, filesize=14.4 K 2024-11-07T14:14:11,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/14d5a92110394ece863ff042c4f8f073 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/14d5a92110394ece863ff042c4f8f073 2024-11-07T14:14:11,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T14:14:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T14:14:11,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/14d5a92110394ece863ff042c4f8f073, entries=150, sequenceid=452, filesize=12.0 K 2024-11-07T14:14:11,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/d8464a5ac12f41dcbe64dd345bc16681 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d8464a5ac12f41dcbe64dd345bc16681 2024-11-07T14:14:11,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d8464a5ac12f41dcbe64dd345bc16681, entries=150, sequenceid=452, filesize=12.0 K 2024-11-07T14:14:11,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8133e138f01b5d4b631e25ead6cfe8cd in 892ms, sequenceid=452, compaction requested=true 2024-11-07T14:14:11,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:11,690 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T14:14:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:11,691 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T14:14:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:11,694 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62425 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T14:14:11,694 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:11,694 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,694 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f7ec1271cd6147639e026ac35fc0a9c8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/0a6edc35144a479cb9720295edbdf19f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4d1dd28812994d70a6893692fc611f55, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/d0258f31f6184b7b979be4146e97abc2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/14d5a92110394ece863ff042c4f8f073] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=61.0 K 2024-11-07T14:14:11,694 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67305 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T14:14:11,694 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:11,695 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,695 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9be20dc3d2a14fa7941f8b89dda2000e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9ccc75344d914ea58e114feac03d32d1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/699e479f1dfe4d77bcdb2b1b242d6ce6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2d76e9c0c10c42cfbf8912c23165830d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/fe1bafbd9df6437e8d3441bfb8ef15eb] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=65.7 K 2024-11-07T14:14:11,695 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9be20dc3d2a14fa7941f8b89dda2000e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988847016 2024-11-07T14:14:11,696 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ec1271cd6147639e026ac35fc0a9c8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988847016 2024-11-07T14:14:11,696 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6edc35144a479cb9720295edbdf19f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730988847746 2024-11-07T14:14:11,696 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ccc75344d914ea58e114feac03d32d1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730988847746 2024-11-07T14:14:11,697 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1dd28812994d70a6893692fc611f55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1730988847893 2024-11-07T14:14:11,697 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 699e479f1dfe4d77bcdb2b1b242d6ce6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1730988847893 2024-11-07T14:14:11,697 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d0258f31f6184b7b979be4146e97abc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1730988848581 2024-11-07T14:14:11,698 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d76e9c0c10c42cfbf8912c23165830d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1730988848581 2024-11-07T14:14:11,698 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d5a92110394ece863ff042c4f8f073, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1730988850791 2024-11-07T14:14:11,699 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1bafbd9df6437e8d3441bfb8ef15eb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1730988850791 2024-11-07T14:14:11,722 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#95 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:11,723 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/c99a92ec91734980a138154d72e9d74d is 50, key is test_row_0/A:col10/1730988850791/Put/seqid=0 2024-11-07T14:14:11,725 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:11,726 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/3568f3f41bbe45169fbe4a9014ffb80f is 50, key is test_row_0/B:col10/1730988850791/Put/seqid=0 2024-11-07T14:14:11,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741932_1108 (size=13391) 2024-11-07T14:14:11,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741933_1109 (size=13391) 2024-11-07T14:14:11,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T14:14:11,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:11,818 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:14:11,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/734faa5f496241cc8a46767e5094b71f is 50, key is test_row_0/A:col10/1730988850853/Put/seqid=0 2024-11-07T14:14:11,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741934_1110 (size=12301) 2024-11-07T14:14:12,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. as already flushing 2024-11-07T14:14:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988912023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988912024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988912132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988912132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,144 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/c99a92ec91734980a138154d72e9d74d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c99a92ec91734980a138154d72e9d74d 2024-11-07T14:14:12,144 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/3568f3f41bbe45169fbe4a9014ffb80f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3568f3f41bbe45169fbe4a9014ffb80f 2024-11-07T14:14:12,160 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into c99a92ec91734980a138154d72e9d74d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:12,160 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:12,160 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=11, startTime=1730988851690; duration=0sec 2024-11-07T14:14:12,161 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:12,161 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:12,161 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T14:14:12,168 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62391 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T14:14:12,168 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into 3568f3f41bbe45169fbe4a9014ffb80f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:12,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:12,169 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:14:12,169 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=11, startTime=1730988851691; duration=0sec 2024-11-07T14:14:12,169 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:12,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:12,169 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/3296a267dd7941738608bceb9b699ee1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c6b683d784494cc49ab20f42c78578d4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/370e9a22af2f4e33a718d6088d6bb441, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/afaf19406e324030960613932e64b24e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d8464a5ac12f41dcbe64dd345bc16681] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=60.9 K 2024-11-07T14:14:12,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:12,170 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3296a267dd7941738608bceb9b699ee1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988847016 2024-11-07T14:14:12,171 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b683d784494cc49ab20f42c78578d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1730988847746 2024-11-07T14:14:12,172 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 370e9a22af2f4e33a718d6088d6bb441, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1730988847893 2024-11-07T14:14:12,173 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting afaf19406e324030960613932e64b24e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1730988848581 2024-11-07T14:14:12,173 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8464a5ac12f41dcbe64dd345bc16681, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1730988850791 2024-11-07T14:14:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T14:14:12,187 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:12,188 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/86e9a2b0081241b2b1ea5bd1565db46b is 50, key is test_row_0/C:col10/1730988850791/Put/seqid=0 2024-11-07T14:14:12,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741935_1111 (size=13357) 2024-11-07T14:14:12,234 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/734faa5f496241cc8a46767e5094b71f 2024-11-07T14:14:12,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/8e836125df4345b681aac0292f4ea3a6 is 50, key is test_row_0/B:col10/1730988850853/Put/seqid=0 2024-11-07T14:14:12,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741936_1112 (size=12301) 2024-11-07T14:14:12,271 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/8e836125df4345b681aac0292f4ea3a6 2024-11-07T14:14:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/9032b25d4e084ac3a673ab8610cd7e6b is 50, key is test_row_0/C:col10/1730988850853/Put/seqid=0 2024-11-07T14:14:12,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741937_1113 (size=12301) 2024-11-07T14:14:12,319 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/9032b25d4e084ac3a673ab8610cd7e6b 2024-11-07T14:14:12,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/734faa5f496241cc8a46767e5094b71f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/734faa5f496241cc8a46767e5094b71f 2024-11-07T14:14:12,341 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/734faa5f496241cc8a46767e5094b71f, entries=150, sequenceid=477, filesize=12.0 K 2024-11-07T14:14:12,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988912342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988912343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/8e836125df4345b681aac0292f4ea3a6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/8e836125df4345b681aac0292f4ea3a6 2024-11-07T14:14:12,360 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/8e836125df4345b681aac0292f4ea3a6, entries=150, sequenceid=477, filesize=12.0 K 2024-11-07T14:14:12,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/9032b25d4e084ac3a673ab8610cd7e6b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9032b25d4e084ac3a673ab8610cd7e6b 2024-11-07T14:14:12,369 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9032b25d4e084ac3a673ab8610cd7e6b, entries=150, sequenceid=477, filesize=12.0 K 2024-11-07T14:14:12,371 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8133e138f01b5d4b631e25ead6cfe8cd in 553ms, sequenceid=477, compaction requested=false 2024-11-07T14:14:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:12,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-07T14:14:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-07T14:14:12,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T14:14:12,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3270 sec 2024-11-07T14:14:12,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.3330 sec 2024-11-07T14:14:12,636 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/86e9a2b0081241b2b1ea5bd1565db46b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/86e9a2b0081241b2b1ea5bd1565db46b 2024-11-07T14:14:12,646 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into 86e9a2b0081241b2b1ea5bd1565db46b(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:12,646 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:12,646 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=11, startTime=1730988851691; duration=0sec 2024-11-07T14:14:12,646 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:12,646 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:12,656 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:12,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:14:12,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/b295216e559b4483922dc2ae6fff31ae is 50, key is test_row_0/A:col10/1730988852021/Put/seqid=0 2024-11-07T14:14:12,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741938_1114 (size=17181) 2024-11-07T14:14:12,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/b295216e559b4483922dc2ae6fff31ae 2024-11-07T14:14:12,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/7ea86c3bd7b74cdf93a2c3c527d781e6 is 50, key is test_row_0/B:col10/1730988852021/Put/seqid=0 2024-11-07T14:14:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741939_1115 (size=12301) 2024-11-07T14:14:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988912714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/7ea86c3bd7b74cdf93a2c3c527d781e6 2024-11-07T14:14:12,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988912720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/9e7be3aedc414c13bc451e0d44e9620a is 50, key is test_row_0/C:col10/1730988852021/Put/seqid=0 2024-11-07T14:14:12,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741940_1116 (size=12301) 2024-11-07T14:14:12,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/9e7be3aedc414c13bc451e0d44e9620a 2024-11-07T14:14:12,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/b295216e559b4483922dc2ae6fff31ae as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/b295216e559b4483922dc2ae6fff31ae 2024-11-07T14:14:12,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/b295216e559b4483922dc2ae6fff31ae, entries=250, sequenceid=493, filesize=16.8 K 2024-11-07T14:14:12,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/7ea86c3bd7b74cdf93a2c3c527d781e6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7ea86c3bd7b74cdf93a2c3c527d781e6 2024-11-07T14:14:12,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7ea86c3bd7b74cdf93a2c3c527d781e6, entries=150, sequenceid=493, filesize=12.0 K 2024-11-07T14:14:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/9e7be3aedc414c13bc451e0d44e9620a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9e7be3aedc414c13bc451e0d44e9620a 2024-11-07T14:14:12,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9e7be3aedc414c13bc451e0d44e9620a, entries=150, sequenceid=493, filesize=12.0 K 2024-11-07T14:14:12,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8133e138f01b5d4b631e25ead6cfe8cd in 134ms, sequenceid=493, compaction requested=true 2024-11-07T14:14:12,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:12,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:12,791 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:12,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:12,791 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:12,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:12,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:12,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:12,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:12,792 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:12,792 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:12,793 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:12,793 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:12,793 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:12,793 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:12,793 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3568f3f41bbe45169fbe4a9014ffb80f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/8e836125df4345b681aac0292f4ea3a6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7ea86c3bd7b74cdf93a2c3c527d781e6] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=37.1 K 2024-11-07T14:14:12,793 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c99a92ec91734980a138154d72e9d74d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/734faa5f496241cc8a46767e5094b71f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/b295216e559b4483922dc2ae6fff31ae] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=41.9 K 2024-11-07T14:14:12,793 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 3568f3f41bbe45169fbe4a9014ffb80f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1730988850791 2024-11-07T14:14:12,793 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99a92ec91734980a138154d72e9d74d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1730988850791 2024-11-07T14:14:12,794 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e836125df4345b681aac0292f4ea3a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730988850845 2024-11-07T14:14:12,794 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 734faa5f496241cc8a46767e5094b71f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730988850845 2024-11-07T14:14:12,794 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ea86c3bd7b74cdf93a2c3c527d781e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1730988852021 2024-11-07T14:14:12,794 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting b295216e559b4483922dc2ae6fff31ae, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1730988852015 2024-11-07T14:14:12,807 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:12,808 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/738c066d635241ddb345b15d9e61fdc8 is 50, key is test_row_0/A:col10/1730988852021/Put/seqid=0 2024-11-07T14:14:12,812 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#105 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:12,812 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/b10a178d98464794a8b4668069551b57 is 50, key is test_row_0/B:col10/1730988852021/Put/seqid=0 2024-11-07T14:14:12,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741941_1117 (size=13493) 2024-11-07T14:14:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:12,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:14:12,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741942_1118 (size=13493) 2024-11-07T14:14:12,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/f54652e457244db2a645d18edeba2895 is 50, key is test_row_0/A:col10/1730988852824/Put/seqid=0 2024-11-07T14:14:12,842 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/b10a178d98464794a8b4668069551b57 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b10a178d98464794a8b4668069551b57 2024-11-07T14:14:12,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741943_1119 (size=12301) 2024-11-07T14:14:12,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/f54652e457244db2a645d18edeba2895 2024-11-07T14:14:12,857 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into b10a178d98464794a8b4668069551b57(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:12,857 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:12,857 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=13, startTime=1730988852791; duration=0sec 2024-11-07T14:14:12,857 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:12,857 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:12,857 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:12,860 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:12,860 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:14:12,860 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:12,860 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/86e9a2b0081241b2b1ea5bd1565db46b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9032b25d4e084ac3a673ab8610cd7e6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9e7be3aedc414c13bc451e0d44e9620a] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=37.1 K 2024-11-07T14:14:12,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/bf3831f32df649388088dfc25dc71ed0 is 50, key is test_row_0/B:col10/1730988852824/Put/seqid=0 2024-11-07T14:14:12,863 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e9a2b0081241b2b1ea5bd1565db46b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1730988850791 2024-11-07T14:14:12,864 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9032b25d4e084ac3a673ab8610cd7e6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730988850845 2024-11-07T14:14:12,864 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7be3aedc414c13bc451e0d44e9620a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1730988852021 2024-11-07T14:14:12,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988912857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988912857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,874 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:12,875 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/8c50302f50024ffb8dbccd64ba17adf5 is 50, key is test_row_0/C:col10/1730988852021/Put/seqid=0 2024-11-07T14:14:12,884 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-07T14:14:12,884 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-07T14:14:12,884 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:12,885 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:12,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741944_1120 (size=12301) 2024-11-07T14:14:12,887 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-07T14:14:12,887 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:12,888 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-07T14:14:12,888 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:12,890 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:12,890 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:12,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741945_1121 (size=13459) 2024-11-07T14:14:12,890 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:12,891 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:12,891 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:12,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:12,891 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:12,891 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:12,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:12,892 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:12,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:12,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:12,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988912972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988912972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988913183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988913183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T14:14:13,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:13,224 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/738c066d635241ddb345b15d9e61fdc8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/738c066d635241ddb345b15d9e61fdc8 2024-11-07T14:14:13,229 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into 738c066d635241ddb345b15d9e61fdc8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:13,229 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:13,229 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=13, startTime=1730988852791; duration=0sec 2024-11-07T14:14:13,229 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:13,229 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:13,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/bf3831f32df649388088dfc25dc71ed0 2024-11-07T14:14:13,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/f3c000ee29df46ea92e0bd7dd2dcf440 is 50, key is test_row_0/C:col10/1730988852824/Put/seqid=0 2024-11-07T14:14:13,296 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/8c50302f50024ffb8dbccd64ba17adf5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8c50302f50024ffb8dbccd64ba17adf5 2024-11-07T14:14:13,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741946_1122 (size=12301) 2024-11-07T14:14:13,302 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into 8c50302f50024ffb8dbccd64ba17adf5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:13,312 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:13,312 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=13, startTime=1730988852791; duration=0sec 2024-11-07T14:14:13,312 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:13,312 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:13,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:13,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40024 deadline: 1730988913492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40018 deadline: 1730988913492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:13,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/f3c000ee29df46ea92e0bd7dd2dcf440 2024-11-07T14:14:13,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/f54652e457244db2a645d18edeba2895 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/f54652e457244db2a645d18edeba2895 2024-11-07T14:14:13,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/f54652e457244db2a645d18edeba2895, entries=150, sequenceid=517, filesize=12.0 K 2024-11-07T14:14:13,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/bf3831f32df649388088dfc25dc71ed0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/bf3831f32df649388088dfc25dc71ed0 2024-11-07T14:14:13,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/bf3831f32df649388088dfc25dc71ed0, entries=150, sequenceid=517, filesize=12.0 K 2024-11-07T14:14:13,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/f3c000ee29df46ea92e0bd7dd2dcf440 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f3c000ee29df46ea92e0bd7dd2dcf440 2024-11-07T14:14:13,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f3c000ee29df46ea92e0bd7dd2dcf440, entries=150, sequenceid=517, filesize=12.0 K 2024-11-07T14:14:13,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 8133e138f01b5d4b631e25ead6cfe8cd in 896ms, sequenceid=517, compaction requested=false 2024-11-07T14:14:13,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:14,005 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-07T14:14:14,005 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-07T14:14:14,005 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:14,005 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:14,005 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:14,005 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:14,005 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:14,005 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:14,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:14,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:14,010 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:14,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T14:14:14,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/9abb439048374fc78f76864e69f0da84 is 50, key is test_row_0/A:col10/1730988852851/Put/seqid=0 2024-11-07T14:14:14,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741947_1123 (size=12301) 2024-11-07T14:14:14,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/9abb439048374fc78f76864e69f0da84 2024-11-07T14:14:14,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/3c3d00e12b154438b268f6b62690049f is 50, key is test_row_0/B:col10/1730988852851/Put/seqid=0 2024-11-07T14:14:14,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741948_1124 (size=12301) 2024-11-07T14:14:14,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/3c3d00e12b154438b268f6b62690049f 2024-11-07T14:14:14,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/4020a895c54d4c4dad76431a09a270e5 is 50, key is test_row_0/C:col10/1730988852851/Put/seqid=0 2024-11-07T14:14:14,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741949_1125 (size=12301) 2024-11-07T14:14:14,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/4020a895c54d4c4dad76431a09a270e5 2024-11-07T14:14:14,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/9abb439048374fc78f76864e69f0da84 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9abb439048374fc78f76864e69f0da84 2024-11-07T14:14:14,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9abb439048374fc78f76864e69f0da84, entries=150, sequenceid=535, filesize=12.0 K 2024-11-07T14:14:14,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/3c3d00e12b154438b268f6b62690049f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3c3d00e12b154438b268f6b62690049f 2024-11-07T14:14:14,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3c3d00e12b154438b268f6b62690049f, entries=150, sequenceid=535, filesize=12.0 K 2024-11-07T14:14:14,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/4020a895c54d4c4dad76431a09a270e5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4020a895c54d4c4dad76431a09a270e5 2024-11-07T14:14:14,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4020a895c54d4c4dad76431a09a270e5, entries=150, sequenceid=535, filesize=12.0 K 2024-11-07T14:14:14,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 8133e138f01b5d4b631e25ead6cfe8cd in 535ms, sequenceid=535, compaction requested=true 2024-11-07T14:14:14,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:14,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:14,547 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:14,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:14,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:14,547 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:14,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:14,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8133e138f01b5d4b631e25ead6cfe8cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:14,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:14,548 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:14,548 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:14,548 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/B is initiating minor compaction (all files) 2024-11-07T14:14:14,548 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/A is initiating minor compaction (all files) 2024-11-07T14:14:14,548 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/B in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:14,548 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/A in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:14,549 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/738c066d635241ddb345b15d9e61fdc8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/f54652e457244db2a645d18edeba2895, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9abb439048374fc78f76864e69f0da84] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=37.2 K 2024-11-07T14:14:14,549 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b10a178d98464794a8b4668069551b57, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/bf3831f32df649388088dfc25dc71ed0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3c3d00e12b154438b268f6b62690049f] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=37.2 K 2024-11-07T14:14:14,549 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738c066d635241ddb345b15d9e61fdc8, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1730988852021 2024-11-07T14:14:14,549 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b10a178d98464794a8b4668069551b57, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1730988852021 2024-11-07T14:14:14,549 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3831f32df649388088dfc25dc71ed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730988852699 2024-11-07T14:14:14,549 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54652e457244db2a645d18edeba2895, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730988852699 2024-11-07T14:14:14,550 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3d00e12b154438b268f6b62690049f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1730988852851 2024-11-07T14:14:14,550 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9abb439048374fc78f76864e69f0da84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1730988852851 2024-11-07T14:14:14,563 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#B#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:14,564 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#A#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:14,564 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/2f4ad6140d7c45fe94633eb315e4b072 is 50, key is test_row_0/B:col10/1730988852851/Put/seqid=0 2024-11-07T14:14:14,564 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/dfcbc60e39b94a01ae4b5a2b0164ca19 is 50, key is test_row_0/A:col10/1730988852851/Put/seqid=0 2024-11-07T14:14:14,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741950_1126 (size=13595) 2024-11-07T14:14:14,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741951_1127 (size=13595) 2024-11-07T14:14:14,581 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/dfcbc60e39b94a01ae4b5a2b0164ca19 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dfcbc60e39b94a01ae4b5a2b0164ca19 2024-11-07T14:14:14,588 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/A of 8133e138f01b5d4b631e25ead6cfe8cd into dfcbc60e39b94a01ae4b5a2b0164ca19(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:14,588 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:14,588 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/A, priority=13, startTime=1730988854547; duration=0sec 2024-11-07T14:14:14,588 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:14,588 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:A 2024-11-07T14:14:14,588 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:14,589 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:14,590 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 8133e138f01b5d4b631e25ead6cfe8cd/C is initiating minor compaction (all files) 2024-11-07T14:14:14,590 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8133e138f01b5d4b631e25ead6cfe8cd/C in TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:14,590 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8c50302f50024ffb8dbccd64ba17adf5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f3c000ee29df46ea92e0bd7dd2dcf440, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4020a895c54d4c4dad76431a09a270e5] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp, totalSize=37.2 K 2024-11-07T14:14:14,590 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c50302f50024ffb8dbccd64ba17adf5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1730988852021 2024-11-07T14:14:14,591 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3c000ee29df46ea92e0bd7dd2dcf440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730988852699 2024-11-07T14:14:14,591 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4020a895c54d4c4dad76431a09a270e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1730988852851 2024-11-07T14:14:14,602 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8133e138f01b5d4b631e25ead6cfe8cd#C#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:14,603 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c4d7494d9bc340fb92980e5c520997d7 is 50, key is test_row_0/C:col10/1730988852851/Put/seqid=0 2024-11-07T14:14:14,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741952_1128 (size=13561) 2024-11-07T14:14:14,982 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/2f4ad6140d7c45fe94633eb315e4b072 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2f4ad6140d7c45fe94633eb315e4b072 2024-11-07T14:14:14,989 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/B of 8133e138f01b5d4b631e25ead6cfe8cd into 2f4ad6140d7c45fe94633eb315e4b072(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:14,989 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:14,989 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/B, priority=13, startTime=1730988854547; duration=0sec 2024-11-07T14:14:14,989 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:14,989 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:B 2024-11-07T14:14:15,015 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/c4d7494d9bc340fb92980e5c520997d7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c4d7494d9bc340fb92980e5c520997d7 2024-11-07T14:14:15,022 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8133e138f01b5d4b631e25ead6cfe8cd/C of 8133e138f01b5d4b631e25ead6cfe8cd into c4d7494d9bc340fb92980e5c520997d7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:15,023 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8133e138f01b5d4b631e25ead6cfe8cd: 2024-11-07T14:14:15,023 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., storeName=8133e138f01b5d4b631e25ead6cfe8cd/C, priority=13, startTime=1730988854547; duration=0sec 2024-11-07T14:14:15,023 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:15,023 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8133e138f01b5d4b631e25ead6cfe8cd:C 2024-11-07T14:14:15,076 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:14:18,825 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-07T14:14:18,825 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:18,825 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:18,825 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:18,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:18,855 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-07T14:14:18,855 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:18,855 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:18,855 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:18,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:18,904 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-07T14:14:18,904 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:18,904 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:18,904 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:18,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:18,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T14:14:18,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-07T14:14:18,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-07T14:14:18,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-07T14:14:18,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-07T14:14:18,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 145 2024-11-07T14:14:18,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T14:14:18,904 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2843 2024-11-07T14:14:18,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2769 2024-11-07T14:14:18,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T14:14:18,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1345 2024-11-07T14:14:18,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4022 rows 2024-11-07T14:14:18,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1347 2024-11-07T14:14:18,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4031 rows 2024-11-07T14:14:18,905 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T14:14:18,905 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:14:18,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:18,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:18,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:18,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T14:14:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T14:14:18,917 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988858917"}]},"ts":"1730988858917"} 2024-11-07T14:14:18,919 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T14:14:18,919 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T14:14:18,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:14:18,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, UNASSIGN}] 2024-11-07T14:14:18,926 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, UNASSIGN 2024-11-07T14:14:18,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=8133e138f01b5d4b631e25ead6cfe8cd, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:18,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, UNASSIGN because future has completed 2024-11-07T14:14:18,930 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:14:18,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:14:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T14:14:19,086 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:19,086 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:14:19,087 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 8133e138f01b5d4b631e25ead6cfe8cd, disabling compactions & flushes 2024-11-07T14:14:19,087 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:19,087 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:19,087 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. after waiting 0 ms 2024-11-07T14:14:19,087 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:19,087 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 8133e138f01b5d4b631e25ead6cfe8cd 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T14:14:19,093 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/ab141b69d886405a9a94a3bc0f8ba9ee is 50, key is test_row_0/A:col10/1730988858823/Put/seqid=0 2024-11-07T14:14:19,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741953_1129 (size=12301) 2024-11-07T14:14:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T14:14:19,500 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/ab141b69d886405a9a94a3bc0f8ba9ee 2024-11-07T14:14:19,516 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/1544708ccfa6446881d03c3c60836276 is 50, key is test_row_0/B:col10/1730988858823/Put/seqid=0 2024-11-07T14:14:19,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741954_1130 (size=12301) 2024-11-07T14:14:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T14:14:19,922 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/1544708ccfa6446881d03c3c60836276 2024-11-07T14:14:19,929 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/165411b6e4c0450995639e6a0f145322 is 50, key is test_row_0/C:col10/1730988858823/Put/seqid=0 2024-11-07T14:14:19,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741955_1131 (size=12301) 2024-11-07T14:14:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T14:14:20,334 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/165411b6e4c0450995639e6a0f145322 2024-11-07T14:14:20,342 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/A/ab141b69d886405a9a94a3bc0f8ba9ee as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/ab141b69d886405a9a94a3bc0f8ba9ee 2024-11-07T14:14:20,349 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/ab141b69d886405a9a94a3bc0f8ba9ee, entries=150, sequenceid=544, filesize=12.0 K 2024-11-07T14:14:20,350 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/B/1544708ccfa6446881d03c3c60836276 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1544708ccfa6446881d03c3c60836276 2024-11-07T14:14:20,355 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1544708ccfa6446881d03c3c60836276, entries=150, sequenceid=544, filesize=12.0 K 2024-11-07T14:14:20,356 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/.tmp/C/165411b6e4c0450995639e6a0f145322 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/165411b6e4c0450995639e6a0f145322 2024-11-07T14:14:20,360 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/165411b6e4c0450995639e6a0f145322, entries=150, sequenceid=544, filesize=12.0 K 2024-11-07T14:14:20,361 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 8133e138f01b5d4b631e25ead6cfe8cd in 1274ms, sequenceid=544, compaction requested=false 2024-11-07T14:14:20,362 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d70d14f7faae43c6bad178bcecf0bd3d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/479da619c7b340e798052fcf88d16033, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c7ee26285e3141c5a716c977346f35b6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2e898e260aae43639ca11646552629f9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fb0ee9125a24baf9c90f90b0918a2f7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/209448dd382b4dbaba27d18c6e8010c2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/edbbd0a521b04bc9bd6f3b7336aeaae2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fd2d437823c46329a8bc51a4117a85e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d9ead8bee404ea4b89c491be223c9e5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/63d6fdeb0761486e8d044ae704d52673, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/83d4a714af53465e88c0cf90d2893b6d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0bbee4b8b8aa4946a0e15e3254b59123, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d71f45ff45a54c468b589f7d09f5ebad, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/e843b13328064481bfcd1200cff16b38, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/5ac126b1a8b747929fa9e61055c31bf2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dee6c413dd144fa89763c0542dd38bb5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d81ab1bf49244c490f31609b5883788, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/39da152643a0400a870d152688bb809f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/da82efbeefe64625b1a4851ed3ef5321, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7dc73c58eb3245839f270cc093033ffc, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/3b56940063ad440c8a745ef262913221, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7278585dd1f04204a098753932209e91, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/863135e0231a4882afc5f5bb59b2b060, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0fe83a007d3d40d79eeabf6a47e2c25d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/19faa427a471489f9c53a67eb1573148, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/90892e474fa9449bafd68419dc2a7723, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9be20dc3d2a14fa7941f8b89dda2000e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/32826b16ac124888b3fe76af9a123d9d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9ccc75344d914ea58e114feac03d32d1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/699e479f1dfe4d77bcdb2b1b242d6ce6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2d76e9c0c10c42cfbf8912c23165830d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/fe1bafbd9df6437e8d3441bfb8ef15eb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c99a92ec91734980a138154d72e9d74d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/734faa5f496241cc8a46767e5094b71f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/b295216e559b4483922dc2ae6fff31ae, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/738c066d635241ddb345b15d9e61fdc8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/f54652e457244db2a645d18edeba2895, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9abb439048374fc78f76864e69f0da84] to archive 2024-11-07T14:14:20,365 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:14:20,370 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d70d14f7faae43c6bad178bcecf0bd3d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d70d14f7faae43c6bad178bcecf0bd3d 2024-11-07T14:14:20,371 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/479da619c7b340e798052fcf88d16033 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/479da619c7b340e798052fcf88d16033 2024-11-07T14:14:20,373 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c7ee26285e3141c5a716c977346f35b6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c7ee26285e3141c5a716c977346f35b6 2024-11-07T14:14:20,374 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2e898e260aae43639ca11646552629f9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2e898e260aae43639ca11646552629f9 2024-11-07T14:14:20,375 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fb0ee9125a24baf9c90f90b0918a2f7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fb0ee9125a24baf9c90f90b0918a2f7 2024-11-07T14:14:20,376 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/209448dd382b4dbaba27d18c6e8010c2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/209448dd382b4dbaba27d18c6e8010c2 2024-11-07T14:14:20,378 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/edbbd0a521b04bc9bd6f3b7336aeaae2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/edbbd0a521b04bc9bd6f3b7336aeaae2 2024-11-07T14:14:20,379 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fd2d437823c46329a8bc51a4117a85e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7fd2d437823c46329a8bc51a4117a85e 2024-11-07T14:14:20,380 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d9ead8bee404ea4b89c491be223c9e5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d9ead8bee404ea4b89c491be223c9e5 2024-11-07T14:14:20,381 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/63d6fdeb0761486e8d044ae704d52673 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/63d6fdeb0761486e8d044ae704d52673 2024-11-07T14:14:20,382 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/83d4a714af53465e88c0cf90d2893b6d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/83d4a714af53465e88c0cf90d2893b6d 2024-11-07T14:14:20,384 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0bbee4b8b8aa4946a0e15e3254b59123 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0bbee4b8b8aa4946a0e15e3254b59123 2024-11-07T14:14:20,385 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d71f45ff45a54c468b589f7d09f5ebad to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/d71f45ff45a54c468b589f7d09f5ebad 2024-11-07T14:14:20,386 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/e843b13328064481bfcd1200cff16b38 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/e843b13328064481bfcd1200cff16b38 2024-11-07T14:14:20,387 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/5ac126b1a8b747929fa9e61055c31bf2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/5ac126b1a8b747929fa9e61055c31bf2 2024-11-07T14:14:20,389 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dee6c413dd144fa89763c0542dd38bb5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dee6c413dd144fa89763c0542dd38bb5 2024-11-07T14:14:20,390 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d81ab1bf49244c490f31609b5883788 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0d81ab1bf49244c490f31609b5883788 2024-11-07T14:14:20,392 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/39da152643a0400a870d152688bb809f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/39da152643a0400a870d152688bb809f 2024-11-07T14:14:20,393 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/da82efbeefe64625b1a4851ed3ef5321 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/da82efbeefe64625b1a4851ed3ef5321 2024-11-07T14:14:20,394 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7dc73c58eb3245839f270cc093033ffc to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7dc73c58eb3245839f270cc093033ffc 2024-11-07T14:14:20,396 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/3b56940063ad440c8a745ef262913221 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/3b56940063ad440c8a745ef262913221 2024-11-07T14:14:20,397 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7278585dd1f04204a098753932209e91 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/7278585dd1f04204a098753932209e91 2024-11-07T14:14:20,398 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/863135e0231a4882afc5f5bb59b2b060 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/863135e0231a4882afc5f5bb59b2b060 2024-11-07T14:14:20,400 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0fe83a007d3d40d79eeabf6a47e2c25d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/0fe83a007d3d40d79eeabf6a47e2c25d 2024-11-07T14:14:20,401 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/19faa427a471489f9c53a67eb1573148 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/19faa427a471489f9c53a67eb1573148 2024-11-07T14:14:20,402 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/90892e474fa9449bafd68419dc2a7723 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/90892e474fa9449bafd68419dc2a7723 2024-11-07T14:14:20,403 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9be20dc3d2a14fa7941f8b89dda2000e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9be20dc3d2a14fa7941f8b89dda2000e 2024-11-07T14:14:20,404 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/32826b16ac124888b3fe76af9a123d9d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/32826b16ac124888b3fe76af9a123d9d 2024-11-07T14:14:20,406 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9ccc75344d914ea58e114feac03d32d1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9ccc75344d914ea58e114feac03d32d1 2024-11-07T14:14:20,407 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/699e479f1dfe4d77bcdb2b1b242d6ce6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/699e479f1dfe4d77bcdb2b1b242d6ce6 2024-11-07T14:14:20,408 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2d76e9c0c10c42cfbf8912c23165830d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/2d76e9c0c10c42cfbf8912c23165830d 2024-11-07T14:14:20,409 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/fe1bafbd9df6437e8d3441bfb8ef15eb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/fe1bafbd9df6437e8d3441bfb8ef15eb 2024-11-07T14:14:20,410 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c99a92ec91734980a138154d72e9d74d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/c99a92ec91734980a138154d72e9d74d 2024-11-07T14:14:20,411 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/734faa5f496241cc8a46767e5094b71f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/734faa5f496241cc8a46767e5094b71f 2024-11-07T14:14:20,412 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/b295216e559b4483922dc2ae6fff31ae to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/b295216e559b4483922dc2ae6fff31ae 2024-11-07T14:14:20,412 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/738c066d635241ddb345b15d9e61fdc8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/738c066d635241ddb345b15d9e61fdc8 2024-11-07T14:14:20,413 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/f54652e457244db2a645d18edeba2895 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/f54652e457244db2a645d18edeba2895 2024-11-07T14:14:20,414 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9abb439048374fc78f76864e69f0da84 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/9abb439048374fc78f76864e69f0da84 2024-11-07T14:14:20,427 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3b657b4544b444a682ca9c97e0a8e52d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/a823d02ada1443d18885d00750ddcbe3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/754601f4616d4d279e467bafe47c51a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/81069e70ac2f4785b58ce5332c53717c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6138ee70655e491da173dc5a15e16942, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/df5f05275692478ab7ddedb50d18f2c8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1178f28d31944b618af71560d525ad43, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/92bdfbd5f396414e888cce2d8da58e27, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4415334790a24839b56ebc6c58cfcbb1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f28921fed8614383b04283fe499f9bb5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b5fd8c36a22c452ca13beecbbc4df497, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/64bd845e304d4800b327f62df9ab52f5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/48e48f44638646feb57b7b43cbb5ef10, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1669080b00d944aaa76aa878cda12a6a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/dbfd3b6a66f44504950f758cd129a3e2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6d5e04d3cd73453da11bdf16170ddbd5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/38bf5129872b4b3284cab0bcb941654d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e4c404e23eb54d7682f34936c54d8da3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e46910b58abe49548ea158939c1dd552, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7956ac91fe1341bd9fee0e2327465f27, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/5f6e143da63b4c74b3fab6be8f18f9b9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6dd6217babfd4000b012c6abda09e6d3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/44299a082c7c420aa5e4c2839cb31e52, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2d236fac82af470282360f6c20d4277f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/c287934e5b854be2b0be257fa93a3e77, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/ba523c2240194b9486205359d1c01b9a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f7ec1271cd6147639e026ac35fc0a9c8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/98ae7707f44340bcab663d5a1ca2101c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/0a6edc35144a479cb9720295edbdf19f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4d1dd28812994d70a6893692fc611f55, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/d0258f31f6184b7b979be4146e97abc2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3568f3f41bbe45169fbe4a9014ffb80f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/14d5a92110394ece863ff042c4f8f073, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/8e836125df4345b681aac0292f4ea3a6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b10a178d98464794a8b4668069551b57, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7ea86c3bd7b74cdf93a2c3c527d781e6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/bf3831f32df649388088dfc25dc71ed0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3c3d00e12b154438b268f6b62690049f] to archive 2024-11-07T14:14:20,428 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:14:20,430 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3b657b4544b444a682ca9c97e0a8e52d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3b657b4544b444a682ca9c97e0a8e52d 2024-11-07T14:14:20,431 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/a823d02ada1443d18885d00750ddcbe3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/a823d02ada1443d18885d00750ddcbe3 2024-11-07T14:14:20,432 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/754601f4616d4d279e467bafe47c51a7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/754601f4616d4d279e467bafe47c51a7 2024-11-07T14:14:20,433 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/81069e70ac2f4785b58ce5332c53717c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/81069e70ac2f4785b58ce5332c53717c 2024-11-07T14:14:20,435 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6138ee70655e491da173dc5a15e16942 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6138ee70655e491da173dc5a15e16942 2024-11-07T14:14:20,436 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/df5f05275692478ab7ddedb50d18f2c8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/df5f05275692478ab7ddedb50d18f2c8 2024-11-07T14:14:20,437 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1178f28d31944b618af71560d525ad43 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1178f28d31944b618af71560d525ad43 2024-11-07T14:14:20,438 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/92bdfbd5f396414e888cce2d8da58e27 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/92bdfbd5f396414e888cce2d8da58e27 2024-11-07T14:14:20,440 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4415334790a24839b56ebc6c58cfcbb1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4415334790a24839b56ebc6c58cfcbb1 2024-11-07T14:14:20,441 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f28921fed8614383b04283fe499f9bb5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f28921fed8614383b04283fe499f9bb5 2024-11-07T14:14:20,442 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b5fd8c36a22c452ca13beecbbc4df497 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b5fd8c36a22c452ca13beecbbc4df497 2024-11-07T14:14:20,443 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/64bd845e304d4800b327f62df9ab52f5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/64bd845e304d4800b327f62df9ab52f5 2024-11-07T14:14:20,444 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/48e48f44638646feb57b7b43cbb5ef10 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/48e48f44638646feb57b7b43cbb5ef10 2024-11-07T14:14:20,445 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1669080b00d944aaa76aa878cda12a6a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1669080b00d944aaa76aa878cda12a6a 2024-11-07T14:14:20,446 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/dbfd3b6a66f44504950f758cd129a3e2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/dbfd3b6a66f44504950f758cd129a3e2 2024-11-07T14:14:20,447 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6d5e04d3cd73453da11bdf16170ddbd5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6d5e04d3cd73453da11bdf16170ddbd5 2024-11-07T14:14:20,448 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/38bf5129872b4b3284cab0bcb941654d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/38bf5129872b4b3284cab0bcb941654d 2024-11-07T14:14:20,450 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e4c404e23eb54d7682f34936c54d8da3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e4c404e23eb54d7682f34936c54d8da3 2024-11-07T14:14:20,451 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e46910b58abe49548ea158939c1dd552 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/e46910b58abe49548ea158939c1dd552 2024-11-07T14:14:20,452 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7956ac91fe1341bd9fee0e2327465f27 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7956ac91fe1341bd9fee0e2327465f27 2024-11-07T14:14:20,453 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/5f6e143da63b4c74b3fab6be8f18f9b9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/5f6e143da63b4c74b3fab6be8f18f9b9 2024-11-07T14:14:20,454 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6dd6217babfd4000b012c6abda09e6d3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/6dd6217babfd4000b012c6abda09e6d3 2024-11-07T14:14:20,455 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/44299a082c7c420aa5e4c2839cb31e52 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/44299a082c7c420aa5e4c2839cb31e52 2024-11-07T14:14:20,456 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2d236fac82af470282360f6c20d4277f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2d236fac82af470282360f6c20d4277f 2024-11-07T14:14:20,457 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/c287934e5b854be2b0be257fa93a3e77 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/c287934e5b854be2b0be257fa93a3e77 2024-11-07T14:14:20,458 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/ba523c2240194b9486205359d1c01b9a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/ba523c2240194b9486205359d1c01b9a 2024-11-07T14:14:20,459 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f7ec1271cd6147639e026ac35fc0a9c8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/f7ec1271cd6147639e026ac35fc0a9c8 2024-11-07T14:14:20,460 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/98ae7707f44340bcab663d5a1ca2101c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/98ae7707f44340bcab663d5a1ca2101c 2024-11-07T14:14:20,461 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/0a6edc35144a479cb9720295edbdf19f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/0a6edc35144a479cb9720295edbdf19f 2024-11-07T14:14:20,462 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4d1dd28812994d70a6893692fc611f55 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/4d1dd28812994d70a6893692fc611f55 2024-11-07T14:14:20,463 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/d0258f31f6184b7b979be4146e97abc2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/d0258f31f6184b7b979be4146e97abc2 2024-11-07T14:14:20,464 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3568f3f41bbe45169fbe4a9014ffb80f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3568f3f41bbe45169fbe4a9014ffb80f 2024-11-07T14:14:20,465 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/14d5a92110394ece863ff042c4f8f073 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/14d5a92110394ece863ff042c4f8f073 2024-11-07T14:14:20,466 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/8e836125df4345b681aac0292f4ea3a6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/8e836125df4345b681aac0292f4ea3a6 2024-11-07T14:14:20,467 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b10a178d98464794a8b4668069551b57 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/b10a178d98464794a8b4668069551b57 2024-11-07T14:14:20,468 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7ea86c3bd7b74cdf93a2c3c527d781e6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/7ea86c3bd7b74cdf93a2c3c527d781e6 2024-11-07T14:14:20,469 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/bf3831f32df649388088dfc25dc71ed0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/bf3831f32df649388088dfc25dc71ed0 2024-11-07T14:14:20,471 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3c3d00e12b154438b268f6b62690049f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/3c3d00e12b154438b268f6b62690049f 2024-11-07T14:14:20,473 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e436103af5384fdab858395677e2daeb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c44df8fa5fdb4d64873bdf5b126dc2a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6685dbd7aff5478ea982e7cc622d1b86, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/5dfe4826a3dd4a619d9cf3f4f5ea667f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b5736d3f07184cffb1dfb3bacacf9c31, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8bd3810980d94a139a0253f860934967, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/aa61c5995e9b4913939299a5832e22e6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/79e8b1addf054065bad1594852787559, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c422385a2fe548fd9216600cde38421b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f86066ba1a09469987ed9b30e106f02b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e5505ed5c39e49c8a93388db641760ab, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0f9d932c80fd413a99431d570e9777d4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6f0bd571e4e541dba05ee07329c5d53b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c76b992c865b4ba995b1f8a5f2a5d488, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/2f888dc8e5ee42798328f0abeeec800c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fb94301e89d34ffaa42bccccb62b3c9b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d2294c9c5032436fb562c5bf85df2e0d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4de5d2bf86ac4227b4f15f3d7595051b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c81a20c1a3e84fb89051f2e12eedb1bd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6cc96d7604ec4bc5b17742c53caed9c0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d917b6d7409b4df499ceab5ad9a3d304, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fd97fc0a584446e29efbed07efef2735, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/46d4e38fc20b44d3845d0589a784e28a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0b4f981dcc40427bb995201826033163, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b669da45da104ba3a40e6fca320a6973, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/3296a267dd7941738608bceb9b699ee1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c144ca4eee7c42afa9f147f67e9f8285, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c6b683d784494cc49ab20f42c78578d4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/370e9a22af2f4e33a718d6088d6bb441, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/afaf19406e324030960613932e64b24e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/86e9a2b0081241b2b1ea5bd1565db46b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d8464a5ac12f41dcbe64dd345bc16681, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9032b25d4e084ac3a673ab8610cd7e6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8c50302f50024ffb8dbccd64ba17adf5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9e7be3aedc414c13bc451e0d44e9620a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f3c000ee29df46ea92e0bd7dd2dcf440, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4020a895c54d4c4dad76431a09a270e5] to archive 2024-11-07T14:14:20,474 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:14:20,476 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e436103af5384fdab858395677e2daeb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e436103af5384fdab858395677e2daeb 2024-11-07T14:14:20,477 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c44df8fa5fdb4d64873bdf5b126dc2a7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c44df8fa5fdb4d64873bdf5b126dc2a7 2024-11-07T14:14:20,478 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6685dbd7aff5478ea982e7cc622d1b86 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6685dbd7aff5478ea982e7cc622d1b86 2024-11-07T14:14:20,479 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/5dfe4826a3dd4a619d9cf3f4f5ea667f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/5dfe4826a3dd4a619d9cf3f4f5ea667f 2024-11-07T14:14:20,480 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b5736d3f07184cffb1dfb3bacacf9c31 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b5736d3f07184cffb1dfb3bacacf9c31 2024-11-07T14:14:20,481 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8bd3810980d94a139a0253f860934967 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8bd3810980d94a139a0253f860934967 2024-11-07T14:14:20,482 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/aa61c5995e9b4913939299a5832e22e6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/aa61c5995e9b4913939299a5832e22e6 2024-11-07T14:14:20,483 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/79e8b1addf054065bad1594852787559 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/79e8b1addf054065bad1594852787559 2024-11-07T14:14:20,484 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c422385a2fe548fd9216600cde38421b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c422385a2fe548fd9216600cde38421b 2024-11-07T14:14:20,485 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f86066ba1a09469987ed9b30e106f02b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f86066ba1a09469987ed9b30e106f02b 2024-11-07T14:14:20,486 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e5505ed5c39e49c8a93388db641760ab to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/e5505ed5c39e49c8a93388db641760ab 2024-11-07T14:14:20,487 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0f9d932c80fd413a99431d570e9777d4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0f9d932c80fd413a99431d570e9777d4 2024-11-07T14:14:20,489 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6f0bd571e4e541dba05ee07329c5d53b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6f0bd571e4e541dba05ee07329c5d53b 2024-11-07T14:14:20,490 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c76b992c865b4ba995b1f8a5f2a5d488 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c76b992c865b4ba995b1f8a5f2a5d488 2024-11-07T14:14:20,491 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/2f888dc8e5ee42798328f0abeeec800c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/2f888dc8e5ee42798328f0abeeec800c 2024-11-07T14:14:20,493 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fb94301e89d34ffaa42bccccb62b3c9b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fb94301e89d34ffaa42bccccb62b3c9b 2024-11-07T14:14:20,494 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d2294c9c5032436fb562c5bf85df2e0d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d2294c9c5032436fb562c5bf85df2e0d 2024-11-07T14:14:20,495 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4de5d2bf86ac4227b4f15f3d7595051b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4de5d2bf86ac4227b4f15f3d7595051b 2024-11-07T14:14:20,496 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c81a20c1a3e84fb89051f2e12eedb1bd to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c81a20c1a3e84fb89051f2e12eedb1bd 2024-11-07T14:14:20,497 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6cc96d7604ec4bc5b17742c53caed9c0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/6cc96d7604ec4bc5b17742c53caed9c0 2024-11-07T14:14:20,498 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d917b6d7409b4df499ceab5ad9a3d304 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d917b6d7409b4df499ceab5ad9a3d304 2024-11-07T14:14:20,499 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fd97fc0a584446e29efbed07efef2735 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/fd97fc0a584446e29efbed07efef2735 2024-11-07T14:14:20,500 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/46d4e38fc20b44d3845d0589a784e28a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/46d4e38fc20b44d3845d0589a784e28a 2024-11-07T14:14:20,501 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0b4f981dcc40427bb995201826033163 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/0b4f981dcc40427bb995201826033163 2024-11-07T14:14:20,502 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b669da45da104ba3a40e6fca320a6973 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/b669da45da104ba3a40e6fca320a6973 2024-11-07T14:14:20,503 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/3296a267dd7941738608bceb9b699ee1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/3296a267dd7941738608bceb9b699ee1 2024-11-07T14:14:20,504 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c144ca4eee7c42afa9f147f67e9f8285 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c144ca4eee7c42afa9f147f67e9f8285 2024-11-07T14:14:20,505 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c6b683d784494cc49ab20f42c78578d4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c6b683d784494cc49ab20f42c78578d4 2024-11-07T14:14:20,506 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/370e9a22af2f4e33a718d6088d6bb441 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/370e9a22af2f4e33a718d6088d6bb441 2024-11-07T14:14:20,507 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/afaf19406e324030960613932e64b24e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/afaf19406e324030960613932e64b24e 2024-11-07T14:14:20,508 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/86e9a2b0081241b2b1ea5bd1565db46b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/86e9a2b0081241b2b1ea5bd1565db46b 2024-11-07T14:14:20,509 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d8464a5ac12f41dcbe64dd345bc16681 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/d8464a5ac12f41dcbe64dd345bc16681 2024-11-07T14:14:20,510 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9032b25d4e084ac3a673ab8610cd7e6b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9032b25d4e084ac3a673ab8610cd7e6b 2024-11-07T14:14:20,511 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8c50302f50024ffb8dbccd64ba17adf5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/8c50302f50024ffb8dbccd64ba17adf5 2024-11-07T14:14:20,512 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9e7be3aedc414c13bc451e0d44e9620a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/9e7be3aedc414c13bc451e0d44e9620a 2024-11-07T14:14:20,513 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f3c000ee29df46ea92e0bd7dd2dcf440 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/f3c000ee29df46ea92e0bd7dd2dcf440 2024-11-07T14:14:20,514 DEBUG [StoreCloser-TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4020a895c54d4c4dad76431a09a270e5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/4020a895c54d4c4dad76431a09a270e5 2024-11-07T14:14:20,519 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=1 2024-11-07T14:14:20,520 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. 2024-11-07T14:14:20,520 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 8133e138f01b5d4b631e25ead6cfe8cd: Waiting for close lock at 1730988859087Running coprocessor pre-close hooks at 1730988859087Disabling compacts and flushes for region at 1730988859087Disabling writes for close at 1730988859087Obtaining lock to block concurrent updates at 1730988859088 (+1 ms)Preparing flush snapshotting stores in 8133e138f01b5d4b631e25ead6cfe8cd at 1730988859088Finished memstore snapshotting TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730988859088Flushing stores of TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd. at 1730988859089 (+1 ms)Flushing 8133e138f01b5d4b631e25ead6cfe8cd/A: creating writer at 1730988859089Flushing 8133e138f01b5d4b631e25ead6cfe8cd/A: appending metadata at 1730988859093 (+4 ms)Flushing 8133e138f01b5d4b631e25ead6cfe8cd/A: closing flushed file at 1730988859093Flushing 8133e138f01b5d4b631e25ead6cfe8cd/B: creating writer at 1730988859511 (+418 ms)Flushing 8133e138f01b5d4b631e25ead6cfe8cd/B: appending metadata at 1730988859515 (+4 ms)Flushing 8133e138f01b5d4b631e25ead6cfe8cd/B: closing flushed file at 1730988859515Flushing 8133e138f01b5d4b631e25ead6cfe8cd/C: creating writer at 1730988859926 (+411 ms)Flushing 8133e138f01b5d4b631e25ead6cfe8cd/C: appending metadata at 1730988859929 (+3 ms)Flushing 8133e138f01b5d4b631e25ead6cfe8cd/C: closing flushed file at 1730988859929Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c4855a1: reopening flushed file at 1730988860341 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dfc6a29: reopening flushed file at 1730988860350 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d5e10ad: reopening flushed file at 1730988860355 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 8133e138f01b5d4b631e25ead6cfe8cd in 1274ms, sequenceid=544, compaction requested=false at 1730988860361 (+6 ms)Writing region close event to WAL at 1730988860516 (+155 ms)Running coprocessor post-close hooks at 1730988860520 (+4 ms)Closed at 1730988860520 2024-11-07T14:14:20,522 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:20,523 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=8133e138f01b5d4b631e25ead6cfe8cd, regionState=CLOSED 2024-11-07T14:14:20,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:14:20,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-07T14:14:20,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 8133e138f01b5d4b631e25ead6cfe8cd, server=b7cc9d764de6,37737,1730988826985 in 1.5960 sec 2024-11-07T14:14:20,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-07T14:14:20,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8133e138f01b5d4b631e25ead6cfe8cd, UNASSIGN in 1.6020 sec 2024-11-07T14:14:20,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-07T14:14:20,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6090 sec 2024-11-07T14:14:20,532 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988860532"}]},"ts":"1730988860532"} 2024-11-07T14:14:20,534 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T14:14:20,534 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T14:14:20,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6230 sec 2024-11-07T14:14:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T14:14:21,063 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:21,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T14:14:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:21,071 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-07T14:14:21,073 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:21,078 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:21,083 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/recovered.edits] 2024-11-07T14:14:21,086 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/ab141b69d886405a9a94a3bc0f8ba9ee to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/ab141b69d886405a9a94a3bc0f8ba9ee 2024-11-07T14:14:21,087 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dfcbc60e39b94a01ae4b5a2b0164ca19 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/A/dfcbc60e39b94a01ae4b5a2b0164ca19 2024-11-07T14:14:21,090 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1544708ccfa6446881d03c3c60836276 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/1544708ccfa6446881d03c3c60836276 2024-11-07T14:14:21,092 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2f4ad6140d7c45fe94633eb315e4b072 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/B/2f4ad6140d7c45fe94633eb315e4b072 2024-11-07T14:14:21,094 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/165411b6e4c0450995639e6a0f145322 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/165411b6e4c0450995639e6a0f145322 2024-11-07T14:14:21,095 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c4d7494d9bc340fb92980e5c520997d7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/C/c4d7494d9bc340fb92980e5c520997d7 2024-11-07T14:14:21,097 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/recovered.edits/547.seqid to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd/recovered.edits/547.seqid 2024-11-07T14:14:21,097 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/8133e138f01b5d4b631e25ead6cfe8cd 2024-11-07T14:14:21,097 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T14:14:21,103 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:21,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-07T14:14:21,114 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T14:14:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-07T14:14:21,192 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T14:14:21,194 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:21,194 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T14:14:21,194 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730988861194"}]},"ts":"9223372036854775807"} 2024-11-07T14:14:21,198 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T14:14:21,198 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8133e138f01b5d4b631e25ead6cfe8cd, NAME => 'TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T14:14:21,198 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T14:14:21,199 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730988861199"}]},"ts":"9223372036854775807"} 2024-11-07T14:14:21,202 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T14:14:21,203 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:21,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 136 msec 2024-11-07T14:14:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-07T14:14:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T14:14:21,394 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:21,407 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1809816429_22 at /127.0.0.1:57188 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b7cc9d764de6:37737-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=634 (was 615) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=246 (was 257) 2024-11-07T14:14:21,418 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=634, ProcessCount=11, AvailableMemoryMB=246 2024-11-07T14:14:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:14:21,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:14:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:21,423 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:14:21,424 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:21,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-07T14:14:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T14:14:21,425 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:14:21,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741956_1132 (size=959) 2024-11-07T14:14:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T14:14:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T14:14:21,835 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 2024-11-07T14:14:21,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741957_1133 (size=53) 2024-11-07T14:14:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T14:14:22,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:14:22,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 76bd866dbccb37465d0501710519b858, disabling compactions & flushes 2024-11-07T14:14:22,248 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:22,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:22,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. after waiting 0 ms 2024-11-07T14:14:22,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:22,248 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:22,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76bd866dbccb37465d0501710519b858: Waiting for close lock at 1730988862247Disabling compacts and flushes for region at 1730988862247Disabling writes for close at 1730988862248 (+1 ms)Writing region close event to WAL at 1730988862248Closed at 1730988862248 2024-11-07T14:14:22,250 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:14:22,250 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730988862250"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730988862250"}]},"ts":"1730988862250"} 2024-11-07T14:14:22,252 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T14:14:22,253 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:14:22,253 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988862253"}]},"ts":"1730988862253"} 2024-11-07T14:14:22,254 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T14:14:22,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, ASSIGN}] 2024-11-07T14:14:22,256 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, ASSIGN 2024-11-07T14:14:22,256 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, ASSIGN; state=OFFLINE, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=false 2024-11-07T14:14:22,407 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=76bd866dbccb37465d0501710519b858, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:22,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, ASSIGN because future has completed 2024-11-07T14:14:22,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:14:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T14:14:22,566 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:22,566 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:14:22,566 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,566 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:14:22,566 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,566 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,568 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,570 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76bd866dbccb37465d0501710519b858 columnFamilyName A 2024-11-07T14:14:22,570 DEBUG [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:22,571 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(327): Store=76bd866dbccb37465d0501710519b858/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:22,571 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,572 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76bd866dbccb37465d0501710519b858 columnFamilyName B 2024-11-07T14:14:22,572 DEBUG [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:22,573 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(327): Store=76bd866dbccb37465d0501710519b858/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:22,573 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,574 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76bd866dbccb37465d0501710519b858 columnFamilyName C 2024-11-07T14:14:22,574 DEBUG [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:22,575 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(327): Store=76bd866dbccb37465d0501710519b858/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:22,575 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,576 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,576 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,577 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,577 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,578 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:14:22,579 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,581 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:14:22,582 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 76bd866dbccb37465d0501710519b858; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71075814, jitterRate=0.05911216139793396}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:14:22,582 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:22,583 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 76bd866dbccb37465d0501710519b858: Running coprocessor pre-open hook at 1730988862567Writing region info on filesystem at 1730988862567Initializing all the Stores at 1730988862567Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988862568 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988862568Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988862568Cleaning up temporary data from old regions at 1730988862577 (+9 ms)Running coprocessor post-open hooks at 1730988862582 (+5 ms)Region opened successfully at 1730988862583 (+1 ms) 2024-11-07T14:14:22,584 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., pid=36, masterSystemTime=1730988862562 2024-11-07T14:14:22,586 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:22,586 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:22,586 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=76bd866dbccb37465d0501710519b858, regionState=OPEN, openSeqNum=2, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:22,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:14:22,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-07T14:14:22,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 in 179 msec 2024-11-07T14:14:22,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-07T14:14:22,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, ASSIGN in 336 msec 2024-11-07T14:14:22,593 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:14:22,594 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988862593"}]},"ts":"1730988862593"} 2024-11-07T14:14:22,595 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T14:14:22,596 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:14:22,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-07T14:14:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T14:14:23,575 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:23,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:23,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:23,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:23,578 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c0765e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:23,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:23,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b457b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:23,581 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:23,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:23,585 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T14:14:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b7cc9d764de6,40745,1730988826115 2024-11-07T14:14:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38eca75c 2024-11-07T14:14:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T14:14:23,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T14:14:23,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:14:23,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:14:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:23,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741958_1134 (size=995) 2024-11-07T14:14:24,012 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T14:14:24,012 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T14:14:24,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:14:24,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, REOPEN/MOVE}] 2024-11-07T14:14:24,021 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, REOPEN/MOVE 2024-11-07T14:14:24,022 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=76bd866dbccb37465d0501710519b858, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, REOPEN/MOVE because future has completed 2024-11-07T14:14:24,024 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:14:24,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:14:24,177 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,177 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:14:24,177 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 76bd866dbccb37465d0501710519b858, disabling compactions & flushes 2024-11-07T14:14:24,177 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,177 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,177 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. after waiting 0 ms 2024-11-07T14:14:24,177 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,182 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T14:14:24,182 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,182 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 76bd866dbccb37465d0501710519b858: Waiting for close lock at 1730988864177Running coprocessor pre-close hooks at 1730988864177Disabling compacts and flushes for region at 1730988864177Disabling writes for close at 1730988864177Writing region close event to WAL at 1730988864178 (+1 ms)Running coprocessor post-close hooks at 1730988864182 (+4 ms)Closed at 1730988864182 2024-11-07T14:14:24,183 WARN [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 76bd866dbccb37465d0501710519b858 to self. 2024-11-07T14:14:24,185 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,186 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=76bd866dbccb37465d0501710519b858, regionState=CLOSED 2024-11-07T14:14:24,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:14:24,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-07T14:14:24,193 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, REOPEN/MOVE; state=CLOSED, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=true 2024-11-07T14:14:24,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 in 167 msec 2024-11-07T14:14:24,344 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=76bd866dbccb37465d0501710519b858, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, REOPEN/MOVE because future has completed 2024-11-07T14:14:24,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:14:24,503 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,503 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:14:24,504 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,504 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:14:24,504 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,504 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,509 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,515 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76bd866dbccb37465d0501710519b858 columnFamilyName A 2024-11-07T14:14:24,517 DEBUG [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:24,518 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(327): Store=76bd866dbccb37465d0501710519b858/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:24,518 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,519 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76bd866dbccb37465d0501710519b858 columnFamilyName B 2024-11-07T14:14:24,519 DEBUG [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:24,520 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(327): Store=76bd866dbccb37465d0501710519b858/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:24,520 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,520 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76bd866dbccb37465d0501710519b858 columnFamilyName C 2024-11-07T14:14:24,521 DEBUG [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:24,521 INFO [StoreOpener-76bd866dbccb37465d0501710519b858-1 {}] regionserver.HStore(327): Store=76bd866dbccb37465d0501710519b858/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:24,521 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,522 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,523 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,524 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,524 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,524 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:14:24,526 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,526 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 76bd866dbccb37465d0501710519b858; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60252840, jitterRate=-0.10216271877288818}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:14:24,526 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,528 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 76bd866dbccb37465d0501710519b858: Running coprocessor pre-open hook at 1730988864504Writing region info on filesystem at 1730988864504Initializing all the Stores at 1730988864505 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988864505Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988864506 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988864506Cleaning up temporary data from old regions at 1730988864524 (+18 ms)Running coprocessor post-open hooks at 1730988864526 (+2 ms)Region opened successfully at 1730988864527 (+1 ms) 2024-11-07T14:14:24,528 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., pid=41, masterSystemTime=1730988864498 2024-11-07T14:14:24,530 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,530 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,531 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=76bd866dbccb37465d0501710519b858, regionState=OPEN, openSeqNum=5, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:14:24,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-07T14:14:24,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 in 187 msec 2024-11-07T14:14:24,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-07T14:14:24,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, REOPEN/MOVE in 514 msec 2024-11-07T14:14:24,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T14:14:24,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-07T14:14:24,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 945 msec 2024-11-07T14:14:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-07T14:14:24,544 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:24,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,546 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a993f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5418ab67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,550 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcf76b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,553 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a1fa1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,556 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc72007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,558 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34101484, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,560 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31811a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,562 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135200f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,564 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103116ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,566 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421a575a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,569 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c709700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,570 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530c764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,572 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154e2171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,574 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553ff8b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18690ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,576 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62bb56ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,577 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b0e159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e58620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:24,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:24,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@770a15a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:24,581 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ebc8e5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:24,582 DEBUG [Thread-681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,582 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,582 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,582 DEBUG [Thread-685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,583 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,583 DEBUG [Thread-687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,584 DEBUG [Thread-683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,584 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,584 DEBUG [Thread-672 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,584 DEBUG [Thread-672 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-07T14:14:24,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T14:14:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,591 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:24,591 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:24,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:14:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,605 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:24,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:24,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a03eaaf5e5b347869892a21812eee0ab_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988864599/Put/seqid=0 2024-11-07T14:14:24,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988924645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988924645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988924646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988924648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988924649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741959_1135 (size=19474) 2024-11-07T14:14:24,665 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:24,669 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a03eaaf5e5b347869892a21812eee0ab_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a03eaaf5e5b347869892a21812eee0ab_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:24,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/e9f1684ae131459e8c036200b0066355, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:24,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/e9f1684ae131459e8c036200b0066355 is 175, key is test_row_0/A:col10/1730988864599/Put/seqid=0 2024-11-07T14:14:24,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741960_1136 (size=56733) 2024-11-07T14:14:24,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/e9f1684ae131459e8c036200b0066355 2024-11-07T14:14:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T14:14:24,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/80f2ba3676b248dd8db8cf100968560a is 50, key is test_row_0/B:col10/1730988864599/Put/seqid=0 2024-11-07T14:14:24,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741961_1137 (size=12001) 2024-11-07T14:14:24,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T14:14:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:24,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988924762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988924763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988924763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988924763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988924763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T14:14:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:24,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T14:14:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988924972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988924973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988924973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988924982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:24,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988924982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:24,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:24,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T14:14:25,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:25,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/80f2ba3676b248dd8db8cf100968560a 2024-11-07T14:14:25,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d41b45d79035424eac409197bb94c305 is 50, key is test_row_0/C:col10/1730988864599/Put/seqid=0 2024-11-07T14:14:25,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741962_1138 (size=12001) 2024-11-07T14:14:25,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T14:14:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T14:14:25,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988925282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988925283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988925283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988925292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988925292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T14:14:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T14:14:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:25,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d41b45d79035424eac409197bb94c305 2024-11-07T14:14:25,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/e9f1684ae131459e8c036200b0066355 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e9f1684ae131459e8c036200b0066355 2024-11-07T14:14:25,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e9f1684ae131459e8c036200b0066355, entries=300, sequenceid=16, filesize=55.4 K 2024-11-07T14:14:25,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/80f2ba3676b248dd8db8cf100968560a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/80f2ba3676b248dd8db8cf100968560a 2024-11-07T14:14:25,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/80f2ba3676b248dd8db8cf100968560a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T14:14:25,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d41b45d79035424eac409197bb94c305 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d41b45d79035424eac409197bb94c305 2024-11-07T14:14:25,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d41b45d79035424eac409197bb94c305, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T14:14:25,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 76bd866dbccb37465d0501710519b858 in 976ms, sequenceid=16, compaction requested=false 2024-11-07T14:14:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:25,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T14:14:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:25,670 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:14:25,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ea5edb36961e4fd18270993c7bb7bc02_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988864641/Put/seqid=0 2024-11-07T14:14:25,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741963_1139 (size=12154) 2024-11-07T14:14:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T14:14:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:25,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:25,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988925802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988925806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988925806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988925808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988925809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988925923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988925923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988925923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988925923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,062 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:14:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:26,095 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ea5edb36961e4fd18270993c7bb7bc02_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ea5edb36961e4fd18270993c7bb7bc02_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:26,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/d0c167e562084c869ce5d8029230089a, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:26,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/d0c167e562084c869ce5d8029230089a is 175, key is test_row_0/A:col10/1730988864641/Put/seqid=0 2024-11-07T14:14:26,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741964_1140 (size=30955) 2024-11-07T14:14:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988926132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988926133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988926133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988926133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988926443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988926443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988926443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988926443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,528 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/d0c167e562084c869ce5d8029230089a 2024-11-07T14:14:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/90f6433380054332b39e18a928ec3105 is 50, key is test_row_0/B:col10/1730988864641/Put/seqid=0 2024-11-07T14:14:26,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741965_1141 (size=12001) 2024-11-07T14:14:26,544 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/90f6433380054332b39e18a928ec3105 2024-11-07T14:14:26,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/df5db178d1674dceb61af5a76aa47a1d is 50, key is test_row_0/C:col10/1730988864641/Put/seqid=0 2024-11-07T14:14:26,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741966_1142 (size=12001) 2024-11-07T14:14:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T14:14:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988926822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,825 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988926952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988926962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988926963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988926963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:26,966 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/df5db178d1674dceb61af5a76aa47a1d 2024-11-07T14:14:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/d0c167e562084c869ce5d8029230089a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/d0c167e562084c869ce5d8029230089a 2024-11-07T14:14:26,984 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/d0c167e562084c869ce5d8029230089a, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T14:14:26,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/90f6433380054332b39e18a928ec3105 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/90f6433380054332b39e18a928ec3105 2024-11-07T14:14:26,998 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/90f6433380054332b39e18a928ec3105, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T14:14:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/df5db178d1674dceb61af5a76aa47a1d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/df5db178d1674dceb61af5a76aa47a1d 2024-11-07T14:14:27,011 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/df5db178d1674dceb61af5a76aa47a1d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T14:14:27,013 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 76bd866dbccb37465d0501710519b858 in 1342ms, sequenceid=40, compaction requested=false 2024-11-07T14:14:27,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:27,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:27,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-07T14:14:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-07T14:14:27,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T14:14:27,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4240 sec 2024-11-07T14:14:27,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.4370 sec 2024-11-07T14:14:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:27,974 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:27,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T14:14:27,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074389523e90734e33ae40b2956102ae35_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988865806/Put/seqid=0 2024-11-07T14:14:27,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741967_1143 (size=14594) 2024-11-07T14:14:28,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988928005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988928006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988928009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988928010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988928123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988928123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988928124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988928124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988928333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988928333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988928335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988928336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:28,403 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074389523e90734e33ae40b2956102ae35_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074389523e90734e33ae40b2956102ae35_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:28,405 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/fd5813a3696142369d9a4ed093a8f70e, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:28,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/fd5813a3696142369d9a4ed093a8f70e is 175, key is test_row_0/A:col10/1730988865806/Put/seqid=0 2024-11-07T14:14:28,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741968_1144 (size=39549) 2024-11-07T14:14:28,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988928642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988928642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988928642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988928652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T14:14:28,762 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:28,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-07T14:14:28,766 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T14:14:28,768 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:28,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:28,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/fd5813a3696142369d9a4ed093a8f70e 2024-11-07T14:14:28,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/9f6faff359b8409089b42e9bab5c51ce is 50, key is test_row_0/B:col10/1730988865806/Put/seqid=0 2024-11-07T14:14:28,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741969_1145 (size=12001) 2024-11-07T14:14:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988928842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:28,845 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T14:14:28,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T14:14:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:28,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T14:14:29,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T14:14:29,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988929162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988929162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988929163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988929163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:29,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T14:14:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/9f6faff359b8409089b42e9bab5c51ce 2024-11-07T14:14:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c52a82de93864d90a4cbcbdfebe7f8c1 is 50, key is test_row_0/C:col10/1730988865806/Put/seqid=0 2024-11-07T14:14:29,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741970_1146 (size=12001) 2024-11-07T14:14:29,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T14:14:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T14:14:29,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T14:14:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:29,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c52a82de93864d90a4cbcbdfebe7f8c1 2024-11-07T14:14:29,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/fd5813a3696142369d9a4ed093a8f70e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/fd5813a3696142369d9a4ed093a8f70e 2024-11-07T14:14:29,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/fd5813a3696142369d9a4ed093a8f70e, entries=200, sequenceid=54, filesize=38.6 K 2024-11-07T14:14:29,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/9f6faff359b8409089b42e9bab5c51ce as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9f6faff359b8409089b42e9bab5c51ce 2024-11-07T14:14:29,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9f6faff359b8409089b42e9bab5c51ce, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T14:14:29,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c52a82de93864d90a4cbcbdfebe7f8c1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c52a82de93864d90a4cbcbdfebe7f8c1 2024-11-07T14:14:29,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c52a82de93864d90a4cbcbdfebe7f8c1, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T14:14:29,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 76bd866dbccb37465d0501710519b858 in 1700ms, sequenceid=54, compaction requested=true 2024-11-07T14:14:29,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:29,674 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:29,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:29,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:29,675 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:29,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:29,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:29,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:29,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:29,675 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:29,676 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/A is initiating minor compaction (all files) 2024-11-07T14:14:29,676 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/A in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,676 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e9f1684ae131459e8c036200b0066355, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/d0c167e562084c869ce5d8029230089a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/fd5813a3696142369d9a4ed093a8f70e] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=124.3 K 2024-11-07T14:14:29,676 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,677 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e9f1684ae131459e8c036200b0066355, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/d0c167e562084c869ce5d8029230089a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/fd5813a3696142369d9a4ed093a8f70e] 2024-11-07T14:14:29,677 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:29,677 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/B is initiating minor compaction (all files) 2024-11-07T14:14:29,677 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9f1684ae131459e8c036200b0066355, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730988864591 2024-11-07T14:14:29,677 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/B in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,677 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/80f2ba3676b248dd8db8cf100968560a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/90f6433380054332b39e18a928ec3105, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9f6faff359b8409089b42e9bab5c51ce] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=35.2 K 2024-11-07T14:14:29,678 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0c167e562084c869ce5d8029230089a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730988864641 2024-11-07T14:14:29,678 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 80f2ba3676b248dd8db8cf100968560a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730988864599 2024-11-07T14:14:29,678 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f6433380054332b39e18a928ec3105, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730988864641 2024-11-07T14:14:29,679 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd5813a3696142369d9a4ed093a8f70e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730988865800 2024-11-07T14:14:29,680 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f6faff359b8409089b42e9bab5c51ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730988865800 2024-11-07T14:14:29,688 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#B#compaction#128 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:29,688 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/38f2693d4645496aac854c5c6068db1d is 50, key is test_row_0/B:col10/1730988865806/Put/seqid=0 2024-11-07T14:14:29,690 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:29,694 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110774cacb83d3f847698e391147ea8495bc_76bd866dbccb37465d0501710519b858 store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:29,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T14:14:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,696 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T14:14:29,699 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110774cacb83d3f847698e391147ea8495bc_76bd866dbccb37465d0501710519b858, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:29,699 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110774cacb83d3f847698e391147ea8495bc_76bd866dbccb37465d0501710519b858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:29,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741971_1147 (size=12104) 2024-11-07T14:14:29,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107664ae2f71c8c49e1b944dd337863a11c_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988867995/Put/seqid=0 2024-11-07T14:14:29,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741972_1148 (size=4469) 2024-11-07T14:14:29,717 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/38f2693d4645496aac854c5c6068db1d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/38f2693d4645496aac854c5c6068db1d 2024-11-07T14:14:29,730 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/B of 76bd866dbccb37465d0501710519b858 into 38f2693d4645496aac854c5c6068db1d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:29,731 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:29,731 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/B, priority=13, startTime=1730988869675; duration=0sec 2024-11-07T14:14:29,731 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:29,731 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:B 2024-11-07T14:14:29,731 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:29,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741973_1149 (size=12154) 2024-11-07T14:14:29,735 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:29,735 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/C is initiating minor compaction (all files) 2024-11-07T14:14:29,735 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/C in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:29,735 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d41b45d79035424eac409197bb94c305, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/df5db178d1674dceb61af5a76aa47a1d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c52a82de93864d90a4cbcbdfebe7f8c1] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=35.2 K 2024-11-07T14:14:29,736 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d41b45d79035424eac409197bb94c305, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730988864599 2024-11-07T14:14:29,736 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting df5db178d1674dceb61af5a76aa47a1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730988864641 2024-11-07T14:14:29,737 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c52a82de93864d90a4cbcbdfebe7f8c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730988865800 2024-11-07T14:14:29,753 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:29,754 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/450f8e3a0516427598e134f21399f556 is 50, key is test_row_0/C:col10/1730988865806/Put/seqid=0 2024-11-07T14:14:29,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741974_1150 (size=12104) 2024-11-07T14:14:29,817 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/450f8e3a0516427598e134f21399f556 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/450f8e3a0516427598e134f21399f556 2024-11-07T14:14:29,826 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/C of 76bd866dbccb37465d0501710519b858 into 450f8e3a0516427598e134f21399f556(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:29,826 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:29,826 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/C, priority=13, startTime=1730988869675; duration=0sec 2024-11-07T14:14:29,826 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:29,826 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:C 2024-11-07T14:14:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T14:14:30,112 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:30,112 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/e017b74eea184a608a39ac5255047b3d is 175, key is test_row_0/A:col10/1730988865806/Put/seqid=0 2024-11-07T14:14:30,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741975_1151 (size=31058) 2024-11-07T14:14:30,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:30,146 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107664ae2f71c8c49e1b944dd337863a11c_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107664ae2f71c8c49e1b944dd337863a11c_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:30,146 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/e017b74eea184a608a39ac5255047b3d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e017b74eea184a608a39ac5255047b3d 2024-11-07T14:14:30,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/1ec7d5a500094ef28ce0ffdc8790e13b, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:30,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/1ec7d5a500094ef28ce0ffdc8790e13b is 175, key is test_row_0/A:col10/1730988867995/Put/seqid=0 2024-11-07T14:14:30,157 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/A of 76bd866dbccb37465d0501710519b858 into e017b74eea184a608a39ac5255047b3d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:30,157 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:30,157 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/A, priority=13, startTime=1730988869674; duration=0sec 2024-11-07T14:14:30,157 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:30,157 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:A 2024-11-07T14:14:30,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741976_1152 (size=30955) 2024-11-07T14:14:30,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:30,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988930200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988930202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988930207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988930208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988930313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988930313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988930322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988930323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988930522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988930522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988930533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988930533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,570 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/1ec7d5a500094ef28ce0ffdc8790e13b 2024-11-07T14:14:30,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/61baefa55c6f4c048a466f7f9c5eb242 is 50, key is test_row_0/B:col10/1730988867995/Put/seqid=0 2024-11-07T14:14:30,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741977_1153 (size=12001) 2024-11-07T14:14:30,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988930832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988930842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988930842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988930842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T14:14:30,998 INFO [master/b7cc9d764de6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T14:14:30,998 INFO [master/b7cc9d764de6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T14:14:31,007 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/61baefa55c6f4c048a466f7f9c5eb242 2024-11-07T14:14:31,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/93c416a35341490891db2ad081dc58b8 is 50, key is test_row_0/C:col10/1730988867995/Put/seqid=0 2024-11-07T14:14:31,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741978_1154 (size=12001) 2024-11-07T14:14:31,032 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/93c416a35341490891db2ad081dc58b8 2024-11-07T14:14:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/1ec7d5a500094ef28ce0ffdc8790e13b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/1ec7d5a500094ef28ce0ffdc8790e13b 2024-11-07T14:14:31,047 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/1ec7d5a500094ef28ce0ffdc8790e13b, entries=150, sequenceid=76, filesize=30.2 K 2024-11-07T14:14:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/61baefa55c6f4c048a466f7f9c5eb242 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/61baefa55c6f4c048a466f7f9c5eb242 2024-11-07T14:14:31,054 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/61baefa55c6f4c048a466f7f9c5eb242, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T14:14:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/93c416a35341490891db2ad081dc58b8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/93c416a35341490891db2ad081dc58b8 2024-11-07T14:14:31,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/93c416a35341490891db2ad081dc58b8, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T14:14:31,063 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 76bd866dbccb37465d0501710519b858 in 1368ms, sequenceid=76, compaction requested=false 2024-11-07T14:14:31,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:31,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:31,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-07T14:14:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-07T14:14:31,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-07T14:14:31,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2970 sec 2024-11-07T14:14:31,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3040 sec 2024-11-07T14:14:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:31,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:31,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T14:14:31,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074cb4fc598e294511ac7f629984fb9437_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988870196/Put/seqid=0 2024-11-07T14:14:31,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741979_1155 (size=12154) 2024-11-07T14:14:31,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:31,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074cb4fc598e294511ac7f629984fb9437_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074cb4fc598e294511ac7f629984fb9437_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:31,368 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/470199a862344d22960fc5c063c5baa9, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:31,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/470199a862344d22960fc5c063c5baa9 is 175, key is test_row_0/A:col10/1730988870196/Put/seqid=0 2024-11-07T14:14:31,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741980_1156 (size=30955) 2024-11-07T14:14:31,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/470199a862344d22960fc5c063c5baa9 2024-11-07T14:14:31,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988931375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988931378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988931379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988931380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ae6b422c1dec452ea53c369c22b451c7 is 50, key is test_row_0/B:col10/1730988870196/Put/seqid=0 2024-11-07T14:14:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741981_1157 (size=12001) 2024-11-07T14:14:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988931482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988931494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988931494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988931495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988931702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988931702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988931703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988931712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:31,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ae6b422c1dec452ea53c369c22b451c7 2024-11-07T14:14:31,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/ed35784a053946e0a637068107a3cd84 is 50, key is test_row_0/C:col10/1730988870196/Put/seqid=0 2024-11-07T14:14:31,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741982_1158 (size=12001) 2024-11-07T14:14:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988932013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988932013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988932023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988932023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/ed35784a053946e0a637068107a3cd84 2024-11-07T14:14:32,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/470199a862344d22960fc5c063c5baa9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470199a862344d22960fc5c063c5baa9 2024-11-07T14:14:32,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470199a862344d22960fc5c063c5baa9, entries=150, sequenceid=94, filesize=30.2 K 2024-11-07T14:14:32,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ae6b422c1dec452ea53c369c22b451c7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ae6b422c1dec452ea53c369c22b451c7 2024-11-07T14:14:32,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ae6b422c1dec452ea53c369c22b451c7, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T14:14:32,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/ed35784a053946e0a637068107a3cd84 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/ed35784a053946e0a637068107a3cd84 2024-11-07T14:14:32,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/ed35784a053946e0a637068107a3cd84, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T14:14:32,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 76bd866dbccb37465d0501710519b858 in 933ms, sequenceid=94, compaction requested=true 2024-11-07T14:14:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:32,278 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:32,278 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:32,279 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:32,279 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:32,280 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/B is initiating minor compaction (all files) 2024-11-07T14:14:32,280 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/A is initiating minor compaction (all files) 2024-11-07T14:14:32,280 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/A in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:32,280 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/B in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:32,280 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e017b74eea184a608a39ac5255047b3d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/1ec7d5a500094ef28ce0ffdc8790e13b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470199a862344d22960fc5c063c5baa9] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=90.8 K 2024-11-07T14:14:32,280 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/38f2693d4645496aac854c5c6068db1d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/61baefa55c6f4c048a466f7f9c5eb242, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ae6b422c1dec452ea53c369c22b451c7] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=35.3 K 2024-11-07T14:14:32,280 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:32,280 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e017b74eea184a608a39ac5255047b3d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/1ec7d5a500094ef28ce0ffdc8790e13b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470199a862344d22960fc5c063c5baa9] 2024-11-07T14:14:32,281 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f2693d4645496aac854c5c6068db1d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730988865800 2024-11-07T14:14:32,281 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 61baefa55c6f4c048a466f7f9c5eb242, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988867995 2024-11-07T14:14:32,282 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting ae6b422c1dec452ea53c369c22b451c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730988870196 2024-11-07T14:14:32,282 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e017b74eea184a608a39ac5255047b3d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730988865800 2024-11-07T14:14:32,283 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ec7d5a500094ef28ce0ffdc8790e13b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988867995 2024-11-07T14:14:32,284 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 470199a862344d22960fc5c063c5baa9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730988870196 2024-11-07T14:14:32,298 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#B#compaction#137 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:32,299 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/35b3cdf037f54d72bbf74654726e8236 is 50, key is test_row_0/B:col10/1730988870196/Put/seqid=0 2024-11-07T14:14:32,304 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:32,311 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411074800dc61a3ee43c0a859feb2afcd4daa_76bd866dbccb37465d0501710519b858 store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:32,314 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411074800dc61a3ee43c0a859feb2afcd4daa_76bd866dbccb37465d0501710519b858, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:32,315 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074800dc61a3ee43c0a859feb2afcd4daa_76bd866dbccb37465d0501710519b858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:32,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741983_1159 (size=12207) 2024-11-07T14:14:32,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741984_1160 (size=4469) 2024-11-07T14:14:32,332 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#A#compaction#138 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:32,333 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/6e0c6743b10b48cfafd45a6a43296714 is 175, key is test_row_0/A:col10/1730988870196/Put/seqid=0 2024-11-07T14:14:32,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741985_1161 (size=31161) 2024-11-07T14:14:32,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:32,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T14:14:32,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bbf0d75e8efd4b52895424f2607d5cb0_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988871378/Put/seqid=0 2024-11-07T14:14:32,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988932547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988932550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988932550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988932551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741986_1162 (size=14594) 2024-11-07T14:14:32,561 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:32,568 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bbf0d75e8efd4b52895424f2607d5cb0_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bbf0d75e8efd4b52895424f2607d5cb0_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:32,569 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/a4b971da2c6a495781773d7f82d97bce, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:32,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/a4b971da2c6a495781773d7f82d97bce is 175, key is test_row_0/A:col10/1730988871378/Put/seqid=0 2024-11-07T14:14:32,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741987_1163 (size=39549) 2024-11-07T14:14:32,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988932662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988932662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988932663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988932663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,731 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/35b3cdf037f54d72bbf74654726e8236 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/35b3cdf037f54d72bbf74654726e8236 2024-11-07T14:14:32,738 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/B of 76bd866dbccb37465d0501710519b858 into 35b3cdf037f54d72bbf74654726e8236(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:32,738 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:32,738 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/B, priority=13, startTime=1730988872278; duration=0sec 2024-11-07T14:14:32,738 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:32,738 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:B 2024-11-07T14:14:32,738 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:32,740 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:32,740 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/C is initiating minor compaction (all files) 2024-11-07T14:14:32,740 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/C in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:32,740 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/450f8e3a0516427598e134f21399f556, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/93c416a35341490891db2ad081dc58b8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/ed35784a053946e0a637068107a3cd84] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=35.3 K 2024-11-07T14:14:32,741 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 450f8e3a0516427598e134f21399f556, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730988865800 2024-11-07T14:14:32,741 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 93c416a35341490891db2ad081dc58b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988867995 2024-11-07T14:14:32,742 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting ed35784a053946e0a637068107a3cd84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730988870196 2024-11-07T14:14:32,749 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#C#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:32,750 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/09f6276771f24af2b08069ac00f9702e is 50, key is test_row_0/C:col10/1730988870196/Put/seqid=0 2024-11-07T14:14:32,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741988_1164 (size=12207) 2024-11-07T14:14:32,759 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/6e0c6743b10b48cfafd45a6a43296714 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/6e0c6743b10b48cfafd45a6a43296714 2024-11-07T14:14:32,765 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/A of 76bd866dbccb37465d0501710519b858 into 6e0c6743b10b48cfafd45a6a43296714(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:32,766 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:32,766 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/A, priority=13, startTime=1730988872278; duration=0sec 2024-11-07T14:14:32,766 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:32,766 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:A 2024-11-07T14:14:32,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988932862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,864 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988932872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988932872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988932873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988932873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T14:14:32,923 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:32,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-07T14:14:32,926 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T14:14:32,927 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:32,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:32,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/a4b971da2c6a495781773d7f82d97bce 2024-11-07T14:14:32,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/c4066e22375d480a80be1178dc985a88 is 50, key is test_row_0/B:col10/1730988871378/Put/seqid=0 2024-11-07T14:14:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741989_1165 (size=12001) 2024-11-07T14:14:32,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/c4066e22375d480a80be1178dc985a88 2024-11-07T14:14:33,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d892d45613ae4cf4823b037b311a032f is 50, key is test_row_0/C:col10/1730988871378/Put/seqid=0 2024-11-07T14:14:33,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741990_1166 (size=12001) 2024-11-07T14:14:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T14:14:33,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T14:14:33,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:33,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:33,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:33,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,160 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/09f6276771f24af2b08069ac00f9702e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/09f6276771f24af2b08069ac00f9702e 2024-11-07T14:14:33,165 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/C of 76bd866dbccb37465d0501710519b858 into 09f6276771f24af2b08069ac00f9702e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:33,165 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:33,165 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/C, priority=13, startTime=1730988872278; duration=0sec 2024-11-07T14:14:33,165 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:33,165 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:C 2024-11-07T14:14:33,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988933182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988933182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988933182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988933182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T14:14:33,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:33,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T14:14:33,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T14:14:33,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:33,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:33,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:33,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:33,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d892d45613ae4cf4823b037b311a032f 2024-11-07T14:14:33,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/a4b971da2c6a495781773d7f82d97bce as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a4b971da2c6a495781773d7f82d97bce 2024-11-07T14:14:33,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a4b971da2c6a495781773d7f82d97bce, entries=200, sequenceid=117, filesize=38.6 K 2024-11-07T14:14:33,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/c4066e22375d480a80be1178dc985a88 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/c4066e22375d480a80be1178dc985a88 2024-11-07T14:14:33,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/c4066e22375d480a80be1178dc985a88, entries=150, sequenceid=117, filesize=11.7 K 2024-11-07T14:14:33,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d892d45613ae4cf4823b037b311a032f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d892d45613ae4cf4823b037b311a032f 2024-11-07T14:14:33,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d892d45613ae4cf4823b037b311a032f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-07T14:14:33,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 76bd866dbccb37465d0501710519b858 in 896ms, sequenceid=117, compaction requested=false 2024-11-07T14:14:33,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:33,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T14:14:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:33,541 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:14:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T14:14:33,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110739073c30d16b424ba6fa28ee83898e08_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988872548/Put/seqid=0 2024-11-07T14:14:33,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741991_1167 (size=12204) 2024-11-07T14:14:33,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:33,594 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110739073c30d16b424ba6fa28ee83898e08_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110739073c30d16b424ba6fa28ee83898e08_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:33,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/44050868f42040d18e0f46e576719dbe, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:33,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/44050868f42040d18e0f46e576719dbe is 175, key is test_row_0/A:col10/1730988872548/Put/seqid=0 2024-11-07T14:14:33,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741992_1168 (size=31005) 2024-11-07T14:14:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988933732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988933732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988933733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988933735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988933843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988933843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988933843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:33,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988933843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,005 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/44050868f42040d18e0f46e576719dbe 2024-11-07T14:14:34,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/5d95e08a92ce4edebe7056d01f41c015 is 50, key is test_row_0/B:col10/1730988872548/Put/seqid=0 2024-11-07T14:14:34,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741993_1169 (size=12051) 2024-11-07T14:14:34,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988934052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988934052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988934052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988934052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T14:14:34,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988934362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988934362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988934363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988934363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,436 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/5d95e08a92ce4edebe7056d01f41c015 2024-11-07T14:14:34,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c68689b662bc43e69a0ffae9b04f5b1f is 50, key is test_row_0/C:col10/1730988872548/Put/seqid=0 2024-11-07T14:14:34,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741994_1170 (size=12051) 2024-11-07T14:14:34,871 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c68689b662bc43e69a0ffae9b04f5b1f 2024-11-07T14:14:34,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988934872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988934873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988934873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988934873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:34,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/44050868f42040d18e0f46e576719dbe as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/44050868f42040d18e0f46e576719dbe 2024-11-07T14:14:34,888 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/44050868f42040d18e0f46e576719dbe, entries=150, sequenceid=133, filesize=30.3 K 2024-11-07T14:14:34,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/5d95e08a92ce4edebe7056d01f41c015 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/5d95e08a92ce4edebe7056d01f41c015 2024-11-07T14:14:34,906 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/5d95e08a92ce4edebe7056d01f41c015, entries=150, sequenceid=133, filesize=11.8 K 2024-11-07T14:14:34,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c68689b662bc43e69a0ffae9b04f5b1f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c68689b662bc43e69a0ffae9b04f5b1f 2024-11-07T14:14:34,915 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c68689b662bc43e69a0ffae9b04f5b1f, entries=150, sequenceid=133, filesize=11.8 K 2024-11-07T14:14:34,916 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 76bd866dbccb37465d0501710519b858 in 1375ms, sequenceid=133, compaction requested=true 2024-11-07T14:14:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-07T14:14:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-07T14:14:34,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T14:14:34,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9920 sec 2024-11-07T14:14:34,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0030 sec 2024-11-07T14:14:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T14:14:35,072 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:35,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-07T14:14:35,075 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T14:14:35,076 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:35,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T14:14:35,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T14:14:35,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:35,229 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T14:14:35,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110771d9234924a443a6b02baa95dfca637f_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988873733/Put/seqid=0 2024-11-07T14:14:35,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741995_1171 (size=12304) 2024-11-07T14:14:35,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,259 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110771d9234924a443a6b02baa95dfca637f_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110771d9234924a443a6b02baa95dfca637f_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:35,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/58e158f146964309a89ba47011db7cd2, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:35,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/58e158f146964309a89ba47011db7cd2 is 175, key is test_row_0/A:col10/1730988873733/Put/seqid=0 2024-11-07T14:14:35,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741996_1172 (size=31105) 2024-11-07T14:14:35,297 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/58e158f146964309a89ba47011db7cd2 2024-11-07T14:14:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/af137cfb0fbf447ebd834fffb53820ce is 50, key is test_row_0/B:col10/1730988873733/Put/seqid=0 2024-11-07T14:14:35,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741997_1173 (size=12151) 2024-11-07T14:14:35,318 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/af137cfb0fbf447ebd834fffb53820ce 2024-11-07T14:14:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/19867b3fbe8f409d8dbd490c02d9cc2f is 50, key is test_row_0/C:col10/1730988873733/Put/seqid=0 2024-11-07T14:14:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741998_1174 (size=12151) 2024-11-07T14:14:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T14:14:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T14:14:35,753 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/19867b3fbe8f409d8dbd490c02d9cc2f 2024-11-07T14:14:35,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/58e158f146964309a89ba47011db7cd2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/58e158f146964309a89ba47011db7cd2 2024-11-07T14:14:35,767 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/58e158f146964309a89ba47011db7cd2, entries=150, sequenceid=156, filesize=30.4 K 2024-11-07T14:14:35,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/af137cfb0fbf447ebd834fffb53820ce as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/af137cfb0fbf447ebd834fffb53820ce 2024-11-07T14:14:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,774 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/af137cfb0fbf447ebd834fffb53820ce, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T14:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/19867b3fbe8f409d8dbd490c02d9cc2f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/19867b3fbe8f409d8dbd490c02d9cc2f 2024-11-07T14:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,781 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/19867b3fbe8f409d8dbd490c02d9cc2f, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T14:14:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,782 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 76bd866dbccb37465d0501710519b858 in 553ms, sequenceid=156, compaction requested=true 2024-11-07T14:14:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:35,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:35,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-07T14:14:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-07T14:14:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T14:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 708 msec 2024-11-07T14:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 713 msec 2024-11-07T14:14:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:35,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:35,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b4884f4b0f144e8197361044d25550ef_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988875937/Put/seqid=0 2024-11-07T14:14:35,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742000_1176 (size=27248) 2024-11-07T14:14:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988935983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988935984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988935986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:35,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988935987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988936092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988936102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988936102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988936103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T14:14:36,212 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:36,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-07T14:14:36,214 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T14:14:36,215 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:36,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:36,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988936301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988936312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988936312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988936312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T14:14:36,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T14:14:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:36,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:36,379 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b4884f4b0f144e8197361044d25550ef_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b4884f4b0f144e8197361044d25550ef_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:36,381 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/147fea8994f44b379e1bd06cc6193812, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:36,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/147fea8994f44b379e1bd06cc6193812 is 175, key is test_row_0/A:col10/1730988875937/Put/seqid=0 2024-11-07T14:14:36,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741999_1175 (size=83035) 2024-11-07T14:14:36,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/147fea8994f44b379e1bd06cc6193812 2024-11-07T14:14:36,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/2934c9e9a1b14ad6bdabe3f191691f1a is 50, key is test_row_0/B:col10/1730988875937/Put/seqid=0 2024-11-07T14:14:36,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742001_1177 (size=12151) 2024-11-07T14:14:36,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T14:14:36,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:36,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T14:14:36,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988936612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988936622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988936622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988936622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:36,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T14:14:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/2934c9e9a1b14ad6bdabe3f191691f1a 2024-11-07T14:14:36,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/20b2656e311e4044ac4d77aea87ae626 is 50, key is test_row_0/C:col10/1730988875937/Put/seqid=0 2024-11-07T14:14:36,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742002_1178 (size=12151) 2024-11-07T14:14:36,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T14:14:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T14:14:36,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T14:14:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:36,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:37,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988937132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:37,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988937133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:37,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T14:14:37,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:37,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:37,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:37,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:37,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988937133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988937133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:37,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/20b2656e311e4044ac4d77aea87ae626 2024-11-07T14:14:37,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/147fea8994f44b379e1bd06cc6193812 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/147fea8994f44b379e1bd06cc6193812 2024-11-07T14:14:37,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/147fea8994f44b379e1bd06cc6193812, entries=450, sequenceid=168, filesize=81.1 K 2024-11-07T14:14:37,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/2934c9e9a1b14ad6bdabe3f191691f1a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/2934c9e9a1b14ad6bdabe3f191691f1a 2024-11-07T14:14:37,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/2934c9e9a1b14ad6bdabe3f191691f1a, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T14:14:37,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/20b2656e311e4044ac4d77aea87ae626 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/20b2656e311e4044ac4d77aea87ae626 2024-11-07T14:14:37,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/20b2656e311e4044ac4d77aea87ae626, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T14:14:37,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 76bd866dbccb37465d0501710519b858 in 1305ms, sequenceid=168, compaction requested=true 2024-11-07T14:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:37,253 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T14:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:37,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:37,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:37,254 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T14:14:37,255 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 215855 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T14:14:37,255 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60561 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T14:14:37,256 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/A is initiating minor compaction (all files) 2024-11-07T14:14:37,256 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/B is initiating minor compaction (all files) 2024-11-07T14:14:37,256 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/A in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:37,256 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/B in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:37,256 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/6e0c6743b10b48cfafd45a6a43296714, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a4b971da2c6a495781773d7f82d97bce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/44050868f42040d18e0f46e576719dbe, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/58e158f146964309a89ba47011db7cd2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/147fea8994f44b379e1bd06cc6193812] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=210.8 K 2024-11-07T14:14:37,256 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/35b3cdf037f54d72bbf74654726e8236, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/c4066e22375d480a80be1178dc985a88, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/5d95e08a92ce4edebe7056d01f41c015, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/af137cfb0fbf447ebd834fffb53820ce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/2934c9e9a1b14ad6bdabe3f191691f1a] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=59.1 K 2024-11-07T14:14:37,256 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:37,256 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/6e0c6743b10b48cfafd45a6a43296714, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a4b971da2c6a495781773d7f82d97bce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/44050868f42040d18e0f46e576719dbe, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/58e158f146964309a89ba47011db7cd2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/147fea8994f44b379e1bd06cc6193812] 2024-11-07T14:14:37,256 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b3cdf037f54d72bbf74654726e8236, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730988870196 2024-11-07T14:14:37,256 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e0c6743b10b48cfafd45a6a43296714, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730988870196 2024-11-07T14:14:37,257 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b971da2c6a495781773d7f82d97bce, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730988871376 2024-11-07T14:14:37,257 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c4066e22375d480a80be1178dc985a88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730988871377 2024-11-07T14:14:37,257 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d95e08a92ce4edebe7056d01f41c015, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730988872545 2024-11-07T14:14:37,257 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44050868f42040d18e0f46e576719dbe, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730988872545 2024-11-07T14:14:37,258 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting af137cfb0fbf447ebd834fffb53820ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730988873723 2024-11-07T14:14:37,258 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58e158f146964309a89ba47011db7cd2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730988873723 2024-11-07T14:14:37,258 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2934c9e9a1b14ad6bdabe3f191691f1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730988875914 2024-11-07T14:14:37,258 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 147fea8994f44b379e1bd06cc6193812, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730988875909 2024-11-07T14:14:37,270 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#B#compaction#152 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:37,270 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/81b26c1ebac64af49227642f04e85f0f is 50, key is test_row_0/B:col10/1730988875937/Put/seqid=0 2024-11-07T14:14:37,271 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:37,274 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107df2fd6073ce346339d6be4c34b3ab3c7_76bd866dbccb37465d0501710519b858 store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:37,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742003_1179 (size=12527) 2024-11-07T14:14:37,280 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107df2fd6073ce346339d6be4c34b3ab3c7_76bd866dbccb37465d0501710519b858, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:37,280 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107df2fd6073ce346339d6be4c34b3ab3c7_76bd866dbccb37465d0501710519b858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:37,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T14:14:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:37,288 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:14:37,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742004_1180 (size=4469) 2024-11-07T14:14:37,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110740eaa80ef4d04f06a9be4be1235c9548_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988875978/Put/seqid=0 2024-11-07T14:14:37,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742005_1181 (size=12304) 2024-11-07T14:14:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T14:14:37,682 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/81b26c1ebac64af49227642f04e85f0f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/81b26c1ebac64af49227642f04e85f0f 2024-11-07T14:14:37,688 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 76bd866dbccb37465d0501710519b858/B of 76bd866dbccb37465d0501710519b858 into 81b26c1ebac64af49227642f04e85f0f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:37,688 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:37,688 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/B, priority=11, startTime=1730988877253; duration=0sec 2024-11-07T14:14:37,689 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:37,689 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:B 2024-11-07T14:14:37,689 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T14:14:37,691 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60561 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T14:14:37,691 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/C is initiating minor compaction (all files) 2024-11-07T14:14:37,691 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/C in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:37,691 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/09f6276771f24af2b08069ac00f9702e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d892d45613ae4cf4823b037b311a032f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c68689b662bc43e69a0ffae9b04f5b1f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/19867b3fbe8f409d8dbd490c02d9cc2f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/20b2656e311e4044ac4d77aea87ae626] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=59.1 K 2024-11-07T14:14:37,691 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 09f6276771f24af2b08069ac00f9702e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730988870196 2024-11-07T14:14:37,692 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d892d45613ae4cf4823b037b311a032f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730988871377 2024-11-07T14:14:37,692 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c68689b662bc43e69a0ffae9b04f5b1f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730988872545 2024-11-07T14:14:37,693 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 19867b3fbe8f409d8dbd490c02d9cc2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730988873723 2024-11-07T14:14:37,693 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b2656e311e4044ac4d77aea87ae626, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730988875914 2024-11-07T14:14:37,693 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:37,694 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/cd60599accdd437ca89e94a49e83f3f9 is 175, key is test_row_0/A:col10/1730988875937/Put/seqid=0 2024-11-07T14:14:37,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742006_1182 (size=31481) 2024-11-07T14:14:37,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:37,710 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110740eaa80ef4d04f06a9be4be1235c9548_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110740eaa80ef4d04f06a9be4be1235c9548_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:37,710 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#C#compaction#155 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:37,710 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/4bc8a840ba97469499c8c65bb43a3486 is 50, key is test_row_0/C:col10/1730988875937/Put/seqid=0 2024-11-07T14:14:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/b047b6891b394c6188529d295335e73d, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/b047b6891b394c6188529d295335e73d is 175, key is test_row_0/A:col10/1730988875978/Put/seqid=0 2024-11-07T14:14:37,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742007_1183 (size=31105) 2024-11-07T14:14:37,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742008_1184 (size=12527) 2024-11-07T14:14:37,725 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/b047b6891b394c6188529d295335e73d 2024-11-07T14:14:37,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/0b80b94f9bd148d89b74a11e6915ccb6 is 50, key is test_row_0/B:col10/1730988875978/Put/seqid=0 2024-11-07T14:14:37,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742009_1185 (size=12151) 2024-11-07T14:14:38,110 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/cd60599accdd437ca89e94a49e83f3f9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/cd60599accdd437ca89e94a49e83f3f9 2024-11-07T14:14:38,119 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 76bd866dbccb37465d0501710519b858/A of 76bd866dbccb37465d0501710519b858 into cd60599accdd437ca89e94a49e83f3f9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:38,119 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:38,119 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/A, priority=11, startTime=1730988877253; duration=0sec 2024-11-07T14:14:38,119 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:38,119 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:A 2024-11-07T14:14:38,132 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/4bc8a840ba97469499c8c65bb43a3486 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/4bc8a840ba97469499c8c65bb43a3486 2024-11-07T14:14:38,138 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/0b80b94f9bd148d89b74a11e6915ccb6 2024-11-07T14:14:38,139 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 76bd866dbccb37465d0501710519b858/C of 76bd866dbccb37465d0501710519b858 into 4bc8a840ba97469499c8c65bb43a3486(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:38,139 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:38,139 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/C, priority=11, startTime=1730988877253; duration=0sec 2024-11-07T14:14:38,139 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:38,139 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:C 2024-11-07T14:14:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:38,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:38,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/f049b2e3a3c0403594d6b3fc318a3c06 is 50, key is test_row_0/C:col10/1730988875978/Put/seqid=0 2024-11-07T14:14:38,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742010_1186 (size=12151) 2024-11-07T14:14:38,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988938160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988938161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988938162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988938162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988938273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988938273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988938273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988938273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T14:14:38,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988938482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988938483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988938483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988938483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,555 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/f049b2e3a3c0403594d6b3fc318a3c06 2024-11-07T14:14:38,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/b047b6891b394c6188529d295335e73d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/b047b6891b394c6188529d295335e73d 2024-11-07T14:14:38,571 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/b047b6891b394c6188529d295335e73d, entries=150, sequenceid=192, filesize=30.4 K 2024-11-07T14:14:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/0b80b94f9bd148d89b74a11e6915ccb6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/0b80b94f9bd148d89b74a11e6915ccb6 2024-11-07T14:14:38,583 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/0b80b94f9bd148d89b74a11e6915ccb6, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T14:14:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/f049b2e3a3c0403594d6b3fc318a3c06 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f049b2e3a3c0403594d6b3fc318a3c06 2024-11-07T14:14:38,592 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f049b2e3a3c0403594d6b3fc318a3c06, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T14:14:38,593 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 76bd866dbccb37465d0501710519b858 in 1305ms, sequenceid=192, compaction requested=false 2024-11-07T14:14:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-07T14:14:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-07T14:14:38,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-07T14:14:38,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3810 sec 2024-11-07T14:14:38,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.3890 sec 2024-11-07T14:14:38,794 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:38,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:14:38,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ef33edd445a8403485d6d412ad5ae136_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988878148/Put/seqid=0 2024-11-07T14:14:38,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742011_1187 (size=12304) 2024-11-07T14:14:38,826 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:38,834 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ef33edd445a8403485d6d412ad5ae136_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ef33edd445a8403485d6d412ad5ae136_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:38,836 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/dbbd639f330d4b18acbe7a715e2dac88, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:38,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/dbbd639f330d4b18acbe7a715e2dac88 is 175, key is test_row_0/A:col10/1730988878148/Put/seqid=0 2024-11-07T14:14:38,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988938836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988938837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742012_1188 (size=31105) 2024-11-07T14:14:38,846 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/dbbd639f330d4b18acbe7a715e2dac88 2024-11-07T14:14:38,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988938842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988938843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/75bda1a208864add8784f2f0676c71a3 is 50, key is test_row_0/B:col10/1730988878148/Put/seqid=0 2024-11-07T14:14:38,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742013_1189 (size=12151) 2024-11-07T14:14:38,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/75bda1a208864add8784f2f0676c71a3 2024-11-07T14:14:38,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/f23609498fa94e19a0c9551e429fe63c is 50, key is test_row_0/C:col10/1730988878148/Put/seqid=0 2024-11-07T14:14:38,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742014_1190 (size=12151) 2024-11-07T14:14:38,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/f23609498fa94e19a0c9551e429fe63c 2024-11-07T14:14:38,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/dbbd639f330d4b18acbe7a715e2dac88 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/dbbd639f330d4b18acbe7a715e2dac88 2024-11-07T14:14:38,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/dbbd639f330d4b18acbe7a715e2dac88, entries=150, sequenceid=208, filesize=30.4 K 2024-11-07T14:14:38,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/75bda1a208864add8784f2f0676c71a3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/75bda1a208864add8784f2f0676c71a3 2024-11-07T14:14:38,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/75bda1a208864add8784f2f0676c71a3, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T14:14:38,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/f23609498fa94e19a0c9551e429fe63c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f23609498fa94e19a0c9551e429fe63c 2024-11-07T14:14:38,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f23609498fa94e19a0c9551e429fe63c, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T14:14:38,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 76bd866dbccb37465d0501710519b858 in 115ms, sequenceid=208, compaction requested=true 2024-11-07T14:14:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:38,909 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:38,909 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:38,910 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:38,910 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/B is initiating minor compaction (all files) 2024-11-07T14:14:38,910 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:38,910 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/B in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:38,910 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/A is initiating minor compaction (all files) 2024-11-07T14:14:38,911 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/A in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:38,911 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/81b26c1ebac64af49227642f04e85f0f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/0b80b94f9bd148d89b74a11e6915ccb6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/75bda1a208864add8784f2f0676c71a3] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=36.0 K 2024-11-07T14:14:38,911 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/cd60599accdd437ca89e94a49e83f3f9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/b047b6891b394c6188529d295335e73d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/dbbd639f330d4b18acbe7a715e2dac88] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=91.5 K 2024-11-07T14:14:38,911 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:38,911 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/cd60599accdd437ca89e94a49e83f3f9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/b047b6891b394c6188529d295335e73d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/dbbd639f330d4b18acbe7a715e2dac88] 2024-11-07T14:14:38,911 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b26c1ebac64af49227642f04e85f0f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730988875914 2024-11-07T14:14:38,911 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd60599accdd437ca89e94a49e83f3f9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730988875914 2024-11-07T14:14:38,911 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b80b94f9bd148d89b74a11e6915ccb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730988875978 2024-11-07T14:14:38,912 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting b047b6891b394c6188529d295335e73d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730988875978 2024-11-07T14:14:38,912 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 75bda1a208864add8784f2f0676c71a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730988878148 2024-11-07T14:14:38,912 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbbd639f330d4b18acbe7a715e2dac88, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730988878148 2024-11-07T14:14:38,919 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:38,920 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#B#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:38,921 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/8412d23a6dc7495fb8dc1758f2ad34a3 is 50, key is test_row_0/B:col10/1730988878148/Put/seqid=0 2024-11-07T14:14:38,923 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411079aab89486e654c12b47adbf32387c03e_76bd866dbccb37465d0501710519b858 store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:38,925 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411079aab89486e654c12b47adbf32387c03e_76bd866dbccb37465d0501710519b858, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:38,925 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079aab89486e654c12b47adbf32387c03e_76bd866dbccb37465d0501710519b858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:38,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742016_1192 (size=4469) 2024-11-07T14:14:38,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742015_1191 (size=12629) 2024-11-07T14:14:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:38,955 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:38,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:14:38,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b31a4dc831c344d0b363d02d692900e1_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988878828/Put/seqid=0 2024-11-07T14:14:38,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988938963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988938965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988938967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988938969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:38,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742017_1193 (size=17284) 2024-11-07T14:14:39,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988939072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988939072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988939083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988939083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988939282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988939282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988939293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988939293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,347 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:39,348 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/7ccdd843755e4f839904df1786ef126d is 175, key is test_row_0/A:col10/1730988878148/Put/seqid=0 2024-11-07T14:14:39,354 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/8412d23a6dc7495fb8dc1758f2ad34a3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8412d23a6dc7495fb8dc1758f2ad34a3 2024-11-07T14:14:39,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742018_1194 (size=31583) 2024-11-07T14:14:39,361 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/B of 76bd866dbccb37465d0501710519b858 into 8412d23a6dc7495fb8dc1758f2ad34a3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:39,361 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:39,361 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/B, priority=13, startTime=1730988878909; duration=0sec 2024-11-07T14:14:39,361 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:39,361 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:B 2024-11-07T14:14:39,361 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:39,362 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:39,362 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/C is initiating minor compaction (all files) 2024-11-07T14:14:39,362 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/C in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:39,362 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/4bc8a840ba97469499c8c65bb43a3486, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f049b2e3a3c0403594d6b3fc318a3c06, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f23609498fa94e19a0c9551e429fe63c] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=36.0 K 2024-11-07T14:14:39,363 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bc8a840ba97469499c8c65bb43a3486, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730988875914 2024-11-07T14:14:39,364 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f049b2e3a3c0403594d6b3fc318a3c06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730988875978 2024-11-07T14:14:39,364 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/7ccdd843755e4f839904df1786ef126d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/7ccdd843755e4f839904df1786ef126d 2024-11-07T14:14:39,364 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f23609498fa94e19a0c9551e429fe63c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730988878148 2024-11-07T14:14:39,370 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/A of 76bd866dbccb37465d0501710519b858 into 7ccdd843755e4f839904df1786ef126d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:39,370 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:39,370 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/A, priority=13, startTime=1730988878909; duration=0sec 2024-11-07T14:14:39,370 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:39,370 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:A 2024-11-07T14:14:39,371 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:39,371 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/f78fb45b4ade4ebba1347722d2281fb3 is 50, key is test_row_0/C:col10/1730988878148/Put/seqid=0 2024-11-07T14:14:39,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742019_1195 (size=12629) 2024-11-07T14:14:39,378 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:39,383 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b31a4dc831c344d0b363d02d692900e1_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b31a4dc831c344d0b363d02d692900e1_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:39,384 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/aea25c51faab4879930a64f8b8ba63fe, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:39,385 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/f78fb45b4ade4ebba1347722d2281fb3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f78fb45b4ade4ebba1347722d2281fb3 2024-11-07T14:14:39,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/aea25c51faab4879930a64f8b8ba63fe is 175, key is test_row_0/A:col10/1730988878828/Put/seqid=0 2024-11-07T14:14:39,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742020_1196 (size=48389) 2024-11-07T14:14:39,395 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76bd866dbccb37465d0501710519b858/C of 76bd866dbccb37465d0501710519b858 into f78fb45b4ade4ebba1347722d2281fb3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:39,395 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:39,395 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/C, priority=13, startTime=1730988878909; duration=0sec 2024-11-07T14:14:39,396 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:39,396 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:C 2024-11-07T14:14:39,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988939592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988939592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988939602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988939602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:39,793 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/aea25c51faab4879930a64f8b8ba63fe 2024-11-07T14:14:39,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/eeace52f66b14c8a872028f69f171b5e is 50, key is test_row_0/B:col10/1730988878828/Put/seqid=0 2024-11-07T14:14:39,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742021_1197 (size=12151) 2024-11-07T14:14:40,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988940102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988940102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988940112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988940112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:40,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/eeace52f66b14c8a872028f69f171b5e 2024-11-07T14:14:40,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/563d0c7449844c18a6d5e70bac88cce7 is 50, key is test_row_0/C:col10/1730988878828/Put/seqid=0 2024-11-07T14:14:40,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742022_1198 (size=12151) 2024-11-07T14:14:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T14:14:40,382 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:40,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-07T14:14:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T14:14:40,385 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:40,386 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:40,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T14:14:40,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T14:14:40,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:40,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:40,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:40,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:40,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:40,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/563d0c7449844c18a6d5e70bac88cce7 2024-11-07T14:14:40,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/aea25c51faab4879930a64f8b8ba63fe as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/aea25c51faab4879930a64f8b8ba63fe 2024-11-07T14:14:40,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/aea25c51faab4879930a64f8b8ba63fe, entries=250, sequenceid=233, filesize=47.3 K 2024-11-07T14:14:40,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/eeace52f66b14c8a872028f69f171b5e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/eeace52f66b14c8a872028f69f171b5e 2024-11-07T14:14:40,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/eeace52f66b14c8a872028f69f171b5e, entries=150, sequenceid=233, filesize=11.9 K 2024-11-07T14:14:40,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/563d0c7449844c18a6d5e70bac88cce7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/563d0c7449844c18a6d5e70bac88cce7 2024-11-07T14:14:40,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/563d0c7449844c18a6d5e70bac88cce7, entries=150, sequenceid=233, filesize=11.9 K 2024-11-07T14:14:40,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 76bd866dbccb37465d0501710519b858 in 1688ms, sequenceid=233, compaction requested=false 2024-11-07T14:14:40,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:40,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T14:14:40,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:40,693 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T14:14:40,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110736ff491c327e45c98402f208611a9d24_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988878958/Put/seqid=0 2024-11-07T14:14:40,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742023_1199 (size=12304) 2024-11-07T14:14:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T14:14:41,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:41,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:41,119 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110736ff491c327e45c98402f208611a9d24_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110736ff491c327e45c98402f208611a9d24_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/ed0aa8aeaecd43ac91d25a4000bc6aa5, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/ed0aa8aeaecd43ac91d25a4000bc6aa5 is 175, key is test_row_0/A:col10/1730988878958/Put/seqid=0 2024-11-07T14:14:41,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742024_1200 (size=31105) 2024-11-07T14:14:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988941152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988941154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988941156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988941157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988941262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988941262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988941272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988941272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988941477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988941477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988941482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988941482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T14:14:41,527 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/ed0aa8aeaecd43ac91d25a4000bc6aa5 2024-11-07T14:14:41,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ce7e4305a7b94f69a53c5cd756c48d75 is 50, key is test_row_0/B:col10/1730988878958/Put/seqid=0 2024-11-07T14:14:41,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742025_1201 (size=12151) 2024-11-07T14:14:41,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988941792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988941792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988941792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988941792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:41,944 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ce7e4305a7b94f69a53c5cd756c48d75 2024-11-07T14:14:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/2c32c7218974434da3d22342f787147b is 50, key is test_row_0/C:col10/1730988878958/Put/seqid=0 2024-11-07T14:14:41,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742026_1202 (size=12151) 2024-11-07T14:14:42,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988942302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988942302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:42,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988942302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988942302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:42,358 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/2c32c7218974434da3d22342f787147b 2024-11-07T14:14:42,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/ed0aa8aeaecd43ac91d25a4000bc6aa5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/ed0aa8aeaecd43ac91d25a4000bc6aa5 2024-11-07T14:14:42,369 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/ed0aa8aeaecd43ac91d25a4000bc6aa5, entries=150, sequenceid=247, filesize=30.4 K 2024-11-07T14:14:42,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ce7e4305a7b94f69a53c5cd756c48d75 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ce7e4305a7b94f69a53c5cd756c48d75 2024-11-07T14:14:42,374 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ce7e4305a7b94f69a53c5cd756c48d75, entries=150, sequenceid=247, filesize=11.9 K 2024-11-07T14:14:42,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/2c32c7218974434da3d22342f787147b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/2c32c7218974434da3d22342f787147b 2024-11-07T14:14:42,380 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/2c32c7218974434da3d22342f787147b, entries=150, sequenceid=247, filesize=11.9 K 2024-11-07T14:14:42,381 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 76bd866dbccb37465d0501710519b858 in 1688ms, sequenceid=247, compaction requested=true 2024-11-07T14:14:42,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:42,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:42,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-07T14:14:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-07T14:14:42,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T14:14:42,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9960 sec 2024-11-07T14:14:42,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.0030 sec 2024-11-07T14:14:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T14:14:42,533 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:42,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-07T14:14:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T14:14:42,536 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:42,536 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:42,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T14:14:42,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T14:14:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:42,689 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:14:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110749deb1684476432bb064a0642a716096_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988881155/Put/seqid=0 2024-11-07T14:14:42,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742027_1203 (size=12454) 2024-11-07T14:14:42,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:42,720 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110749deb1684476432bb064a0642a716096_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749deb1684476432bb064a0642a716096_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/a0ef704ceb8f40a2a62923ad80ded167, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/a0ef704ceb8f40a2a62923ad80ded167 is 175, key is test_row_0/A:col10/1730988881155/Put/seqid=0 2024-11-07T14:14:42,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742028_1204 (size=31255) 2024-11-07T14:14:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T14:14:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:42,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988942927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:43,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988943033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:43,127 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/a0ef704ceb8f40a2a62923ad80ded167 2024-11-07T14:14:43,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/1ca5cc81255c49d99ea60028937216ad is 50, key is test_row_0/B:col10/1730988881155/Put/seqid=0 2024-11-07T14:14:43,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742029_1205 (size=12301) 2024-11-07T14:14:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T14:14:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988943247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:43,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988943322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988943323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988943323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988943323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:43,326 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:43,326 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:43,326 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:43,327 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:43,541 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/1ca5cc81255c49d99ea60028937216ad 2024-11-07T14:14:43,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/8147d1723bc64c47a2eda605a9253d11 is 50, key is test_row_0/C:col10/1730988881155/Put/seqid=0 2024-11-07T14:14:43,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742030_1206 (size=12301) 2024-11-07T14:14:43,562 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/8147d1723bc64c47a2eda605a9253d11 2024-11-07T14:14:43,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988943562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:43,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/a0ef704ceb8f40a2a62923ad80ded167 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a0ef704ceb8f40a2a62923ad80ded167 2024-11-07T14:14:43,580 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a0ef704ceb8f40a2a62923ad80ded167, entries=150, sequenceid=272, filesize=30.5 K 2024-11-07T14:14:43,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/1ca5cc81255c49d99ea60028937216ad as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1ca5cc81255c49d99ea60028937216ad 2024-11-07T14:14:43,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1ca5cc81255c49d99ea60028937216ad, entries=150, sequenceid=272, filesize=12.0 K 2024-11-07T14:14:43,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/8147d1723bc64c47a2eda605a9253d11 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/8147d1723bc64c47a2eda605a9253d11 2024-11-07T14:14:43,597 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/8147d1723bc64c47a2eda605a9253d11, entries=150, sequenceid=272, filesize=12.0 K 2024-11-07T14:14:43,598 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 76bd866dbccb37465d0501710519b858 in 908ms, sequenceid=272, compaction requested=true 2024-11-07T14:14:43,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:43,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:43,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-07T14:14:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-07T14:14:43,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T14:14:43,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-11-07T14:14:43,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.0680 sec 2024-11-07T14:14:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T14:14:43,672 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:43,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-07T14:14:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T14:14:43,675 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:43,675 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:43,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T14:14:43,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-07T14:14:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:43,828 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:43,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079d659a2747fe4deb886e885b9e1c5ff9_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988882918/Put/seqid=0 2024-11-07T14:14:43,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742031_1207 (size=12454) 2024-11-07T14:14:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:43,853 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079d659a2747fe4deb886e885b9e1c5ff9_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079d659a2747fe4deb886e885b9e1c5ff9_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:43,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/0520dc45ca434f62a2e028c7ba3657a2, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/0520dc45ca434f62a2e028c7ba3657a2 is 175, key is test_row_0/A:col10/1730988882918/Put/seqid=0 2024-11-07T14:14:43,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742032_1208 (size=31255) 2024-11-07T14:14:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T14:14:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:44,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. as already flushing 2024-11-07T14:14:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988944191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:44,260 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/0520dc45ca434f62a2e028c7ba3657a2 2024-11-07T14:14:44,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/8e76150ff44e4b048273510a4fa04d0e is 50, key is test_row_0/B:col10/1730988882918/Put/seqid=0 2024-11-07T14:14:44,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742033_1209 (size=12301) 2024-11-07T14:14:44,293 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/8e76150ff44e4b048273510a4fa04d0e 2024-11-07T14:14:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T14:14:44,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988944303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/b938ab6db5c54c76a4668a13ffdca479 is 50, key is test_row_0/C:col10/1730988882918/Put/seqid=0 2024-11-07T14:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:44,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742034_1210 (size=12301) 2024-11-07T14:14:44,333 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/b938ab6db5c54c76a4668a13ffdca479 2024-11-07T14:14:44,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/0520dc45ca434f62a2e028c7ba3657a2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/0520dc45ca434f62a2e028c7ba3657a2 2024-11-07T14:14:44,352 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/0520dc45ca434f62a2e028c7ba3657a2, entries=150, sequenceid=283, filesize=30.5 K 2024-11-07T14:14:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/8e76150ff44e4b048273510a4fa04d0e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8e76150ff44e4b048273510a4fa04d0e 2024-11-07T14:14:44,360 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8e76150ff44e4b048273510a4fa04d0e, entries=150, sequenceid=283, filesize=12.0 K 2024-11-07T14:14:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/b938ab6db5c54c76a4668a13ffdca479 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/b938ab6db5c54c76a4668a13ffdca479 2024-11-07T14:14:44,370 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/b938ab6db5c54c76a4668a13ffdca479, entries=150, sequenceid=283, filesize=12.0 K 2024-11-07T14:14:44,372 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 76bd866dbccb37465d0501710519b858 in 544ms, sequenceid=283, compaction requested=true 2024-11-07T14:14:44,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:44,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:44,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-07T14:14:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-07T14:14:44,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-07T14:14:44,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 699 msec 2024-11-07T14:14:44,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 705 msec 2024-11-07T14:14:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:44,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:44,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T14:14:44,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a723985fd9c1469b9484fd9c0729b520_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988884515/Put/seqid=0 2024-11-07T14:14:44,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742035_1211 (size=14994) 2024-11-07T14:14:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988944561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:44,605 INFO [Thread-685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-685. 2024-11-07T14:14:44,605 DEBUG [Thread-685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:44,605 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:44,606 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:44,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:44,607 INFO [Thread-683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-683. 2024-11-07T14:14:44,607 DEBUG [Thread-683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:44,607 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:44,607 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:44,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:44,607 INFO [Thread-681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-681. 2024-11-07T14:14:44,607 DEBUG [Thread-681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:44,607 DEBUG [Thread-681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:44,607 DEBUG [Thread-681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:44,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:44,615 INFO [Thread-687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-687. 2024-11-07T14:14:44,615 DEBUG [Thread-687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:44,615 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:44,615 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:44,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:44,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988944673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T14:14:44,812 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:44,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988944882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:44,929 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:44,934 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a723985fd9c1469b9484fd9c0729b520_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a723985fd9c1469b9484fd9c0729b520_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:44,935 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/c60dbc08ddcc428ebdaa70dfd1c52ce2, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:44,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/c60dbc08ddcc428ebdaa70dfd1c52ce2 is 175, key is test_row_0/A:col10/1730988884515/Put/seqid=0 2024-11-07T14:14:44,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742036_1212 (size=39949) 2024-11-07T14:14:45,076 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:14:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988945193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60422 deadline: 1730988945332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60442 deadline: 1730988945332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:45,333 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:45,333 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:45,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/c60dbc08ddcc428ebdaa70dfd1c52ce2 2024-11-07T14:14:45,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:45,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1730988945343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60462 deadline: 1730988945343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:45,344 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,344 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 76bd866dbccb37465d0501710519b858 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:14:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:45,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/9ceb9147a5a342e793e70f5fe640ab43 is 50, key is test_row_0/B:col10/1730988884515/Put/seqid=0 2024-11-07T14:14:45,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742037_1213 (size=12301) 2024-11-07T14:14:45,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60410 deadline: 1730988945702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:14:45,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/9ceb9147a5a342e793e70f5fe640ab43 2024-11-07T14:14:45,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c00330b392894a7eab204ee6b0d01722 is 50, key is test_row_0/C:col10/1730988884515/Put/seqid=0 2024-11-07T14:14:45,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742038_1214 (size=12301) 2024-11-07T14:14:46,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c00330b392894a7eab204ee6b0d01722 2024-11-07T14:14:46,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/c60dbc08ddcc428ebdaa70dfd1c52ce2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/c60dbc08ddcc428ebdaa70dfd1c52ce2 2024-11-07T14:14:46,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/c60dbc08ddcc428ebdaa70dfd1c52ce2, entries=200, sequenceid=309, filesize=39.0 K 2024-11-07T14:14:46,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/9ceb9147a5a342e793e70f5fe640ab43 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9ceb9147a5a342e793e70f5fe640ab43 2024-11-07T14:14:46,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9ceb9147a5a342e793e70f5fe640ab43, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T14:14:46,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/c00330b392894a7eab204ee6b0d01722 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c00330b392894a7eab204ee6b0d01722 2024-11-07T14:14:46,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c00330b392894a7eab204ee6b0d01722, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T14:14:46,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 76bd866dbccb37465d0501710519b858 in 1685ms, sequenceid=309, compaction requested=true 2024-11-07T14:14:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:46,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:46,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:46,202 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-07T14:14:46,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:46,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:46,202 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-07T14:14:46,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76bd866dbccb37465d0501710519b858:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:46,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:46,203 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73834 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-07T14:14:46,203 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 213536 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-07T14:14:46,204 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/B is initiating minor compaction (all files) 2024-11-07T14:14:46,204 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/A is initiating minor compaction (all files) 2024-11-07T14:14:46,204 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/B in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:46,204 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/A in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:46,204 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8412d23a6dc7495fb8dc1758f2ad34a3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/eeace52f66b14c8a872028f69f171b5e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ce7e4305a7b94f69a53c5cd756c48d75, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1ca5cc81255c49d99ea60028937216ad, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8e76150ff44e4b048273510a4fa04d0e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9ceb9147a5a342e793e70f5fe640ab43] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=72.1 K 2024-11-07T14:14:46,204 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/7ccdd843755e4f839904df1786ef126d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/aea25c51faab4879930a64f8b8ba63fe, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/ed0aa8aeaecd43ac91d25a4000bc6aa5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a0ef704ceb8f40a2a62923ad80ded167, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/0520dc45ca434f62a2e028c7ba3657a2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/c60dbc08ddcc428ebdaa70dfd1c52ce2] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=208.5 K 2024-11-07T14:14:46,204 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:46,204 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/7ccdd843755e4f839904df1786ef126d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/aea25c51faab4879930a64f8b8ba63fe, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/ed0aa8aeaecd43ac91d25a4000bc6aa5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a0ef704ceb8f40a2a62923ad80ded167, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/0520dc45ca434f62a2e028c7ba3657a2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/c60dbc08ddcc428ebdaa70dfd1c52ce2] 2024-11-07T14:14:46,204 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8412d23a6dc7495fb8dc1758f2ad34a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730988878148 2024-11-07T14:14:46,204 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ccdd843755e4f839904df1786ef126d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730988878148 2024-11-07T14:14:46,205 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting eeace52f66b14c8a872028f69f171b5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1730988878828 2024-11-07T14:14:46,205 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea25c51faab4879930a64f8b8ba63fe, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1730988878828 2024-11-07T14:14:46,205 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7e4305a7b94f69a53c5cd756c48d75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730988878958 2024-11-07T14:14:46,205 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0aa8aeaecd43ac91d25a4000bc6aa5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730988878958 2024-11-07T14:14:46,205 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ca5cc81255c49d99ea60028937216ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1730988881151 2024-11-07T14:14:46,205 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ef704ceb8f40a2a62923ad80ded167, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1730988881151 2024-11-07T14:14:46,205 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e76150ff44e4b048273510a4fa04d0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730988882909 2024-11-07T14:14:46,205 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0520dc45ca434f62a2e028c7ba3657a2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730988882909 2024-11-07T14:14:46,206 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ceb9147a5a342e793e70f5fe640ab43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988884141 2024-11-07T14:14:46,206 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60dbc08ddcc428ebdaa70dfd1c52ce2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988884141 2024-11-07T14:14:46,218 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#B#compaction#179 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:46,219 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/1db3e87935404b8b928d6bc90c7e00fa is 50, key is test_row_0/B:col10/1730988884515/Put/seqid=0 2024-11-07T14:14:46,219 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:46,221 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411070eed56ff97f8435eae9a65e03efbcc0d_76bd866dbccb37465d0501710519b858 store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:46,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742039_1215 (size=12983) 2024-11-07T14:14:46,267 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411070eed56ff97f8435eae9a65e03efbcc0d_76bd866dbccb37465d0501710519b858, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:46,267 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070eed56ff97f8435eae9a65e03efbcc0d_76bd866dbccb37465d0501710519b858 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:46,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742040_1216 (size=4469) 2024-11-07T14:14:46,629 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/1db3e87935404b8b928d6bc90c7e00fa as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1db3e87935404b8b928d6bc90c7e00fa 2024-11-07T14:14:46,635 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 76bd866dbccb37465d0501710519b858/B of 76bd866dbccb37465d0501710519b858 into 1db3e87935404b8b928d6bc90c7e00fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:46,635 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:46,635 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/B, priority=10, startTime=1730988886202; duration=0sec 2024-11-07T14:14:46,635 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:46,635 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:B 2024-11-07T14:14:46,635 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-07T14:14:46,637 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73834 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-07T14:14:46,637 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 76bd866dbccb37465d0501710519b858/C is initiating minor compaction (all files) 2024-11-07T14:14:46,637 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76bd866dbccb37465d0501710519b858/C in TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:46,637 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f78fb45b4ade4ebba1347722d2281fb3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/563d0c7449844c18a6d5e70bac88cce7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/2c32c7218974434da3d22342f787147b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/8147d1723bc64c47a2eda605a9253d11, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/b938ab6db5c54c76a4668a13ffdca479, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c00330b392894a7eab204ee6b0d01722] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp, totalSize=72.1 K 2024-11-07T14:14:46,637 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f78fb45b4ade4ebba1347722d2281fb3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730988878148 2024-11-07T14:14:46,637 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 563d0c7449844c18a6d5e70bac88cce7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1730988878828 2024-11-07T14:14:46,638 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c32c7218974434da3d22342f787147b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730988878958 2024-11-07T14:14:46,638 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8147d1723bc64c47a2eda605a9253d11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1730988881151 2024-11-07T14:14:46,638 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b938ab6db5c54c76a4668a13ffdca479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1730988882909 2024-11-07T14:14:46,639 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c00330b392894a7eab204ee6b0d01722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988884141 2024-11-07T14:14:46,651 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#C#compaction#181 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:46,651 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/136551f463fe48a6858769306ce3e844 is 50, key is test_row_0/C:col10/1730988884515/Put/seqid=0 2024-11-07T14:14:46,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742041_1217 (size=12983) 2024-11-07T14:14:46,673 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76bd866dbccb37465d0501710519b858#A#compaction#180 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:46,673 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/470f5c889d1a49ef963c7f87d54bc86b is 175, key is test_row_0/A:col10/1730988884515/Put/seqid=0 2024-11-07T14:14:46,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742042_1218 (size=31937) 2024-11-07T14:14:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:46,724 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:46,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:46,724 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-11-07T14:14:46,724 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:46,724 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:46,724 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:46,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:46,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075fb882c6431245eda00738a0f5c1fc69_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988884553/Put/seqid=0 2024-11-07T14:14:46,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742043_1219 (size=12454) 2024-11-07T14:14:47,068 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/136551f463fe48a6858769306ce3e844 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/136551f463fe48a6858769306ce3e844 2024-11-07T14:14:47,073 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 76bd866dbccb37465d0501710519b858/C of 76bd866dbccb37465d0501710519b858 into 136551f463fe48a6858769306ce3e844(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:47,073 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:47,073 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/C, priority=10, startTime=1730988886202; duration=0sec 2024-11-07T14:14:47,073 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:47,073 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:C 2024-11-07T14:14:47,082 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/470f5c889d1a49ef963c7f87d54bc86b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470f5c889d1a49ef963c7f87d54bc86b 2024-11-07T14:14:47,088 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 76bd866dbccb37465d0501710519b858/A of 76bd866dbccb37465d0501710519b858 into 470f5c889d1a49ef963c7f87d54bc86b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:47,088 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:47,088 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., storeName=76bd866dbccb37465d0501710519b858/A, priority=10, startTime=1730988886202; duration=0sec 2024-11-07T14:14:47,088 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:47,088 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76bd866dbccb37465d0501710519b858:A 2024-11-07T14:14:47,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:47,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075fb882c6431245eda00738a0f5c1fc69_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075fb882c6431245eda00738a0f5c1fc69_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:47,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/658cca36157446598b1a30568c50030c, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:47,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/658cca36157446598b1a30568c50030c is 175, key is test_row_0/A:col10/1730988884553/Put/seqid=0 2024-11-07T14:14:47,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742044_1220 (size=31255) 2024-11-07T14:14:47,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/658cca36157446598b1a30568c50030c 2024-11-07T14:14:47,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/3f3be19c75bf458ab0d32a3ef582a30e is 50, key is test_row_0/B:col10/1730988884553/Put/seqid=0 2024-11-07T14:14:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742045_1221 (size=12301) 2024-11-07T14:14:47,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/3f3be19c75bf458ab0d32a3ef582a30e 2024-11-07T14:14:47,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d1a08dd9143646449426befc5892303a is 50, key is test_row_0/C:col10/1730988884553/Put/seqid=0 2024-11-07T14:14:47,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742046_1222 (size=12301) 2024-11-07T14:14:48,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d1a08dd9143646449426befc5892303a 2024-11-07T14:14:48,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/658cca36157446598b1a30568c50030c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/658cca36157446598b1a30568c50030c 2024-11-07T14:14:48,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/658cca36157446598b1a30568c50030c, entries=150, sequenceid=321, filesize=30.5 K 2024-11-07T14:14:48,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/3f3be19c75bf458ab0d32a3ef582a30e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/3f3be19c75bf458ab0d32a3ef582a30e 2024-11-07T14:14:48,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/3f3be19c75bf458ab0d32a3ef582a30e, entries=150, sequenceid=321, filesize=12.0 K 2024-11-07T14:14:48,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/d1a08dd9143646449426befc5892303a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d1a08dd9143646449426befc5892303a 2024-11-07T14:14:48,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d1a08dd9143646449426befc5892303a, entries=150, sequenceid=321, filesize=12.0 K 2024-11-07T14:14:48,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 76bd866dbccb37465d0501710519b858 in 1686ms, sequenceid=321, compaction requested=false 2024-11-07T14:14:48,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76bd866dbccb37465d0501710519b858: 2024-11-07T14:14:49,354 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-07T14:14:49,354 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:49,354 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,354 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:49,364 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-07T14:14:49,364 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-07T14:14:49,365 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:49,365 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:49,365 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,365 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,365 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,365 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:49,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:49,383 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-07T14:14:49,383 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:14:49,383 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,383 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3676 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3533 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1595 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4782 rows 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1594 2024-11-07T14:14:49,384 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4779 rows 2024-11-07T14:14:49,384 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T14:14:49,384 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:14:49,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:49,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:14:49,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T14:14:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T14:14:49,388 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988889388"}]},"ts":"1730988889388"} 2024-11-07T14:14:49,389 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T14:14:49,389 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T14:14:49,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:14:49,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, UNASSIGN}] 2024-11-07T14:14:49,392 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, UNASSIGN 2024-11-07T14:14:49,392 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=76bd866dbccb37465d0501710519b858, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:49,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, UNASSIGN because future has completed 2024-11-07T14:14:49,394 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:14:49,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:14:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T14:14:49,546 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:49,546 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:14:49,547 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 76bd866dbccb37465d0501710519b858, disabling compactions & flushes 2024-11-07T14:14:49,547 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:49,547 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:49,547 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. after waiting 0 ms 2024-11-07T14:14:49,547 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:49,547 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 76bd866dbccb37465d0501710519b858 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T14:14:49,555 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fe8ff04fb2844fb99a45703f78bd69ac_76bd866dbccb37465d0501710519b858 is 50, key is test_row_0/A:col10/1730988889382/Put/seqid=0 2024-11-07T14:14:49,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742047_1223 (size=12454) 2024-11-07T14:14:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T14:14:49,960 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:49,965 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fe8ff04fb2844fb99a45703f78bd69ac_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fe8ff04fb2844fb99a45703f78bd69ac_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:49,966 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/87d53f9f3b9d4ab0a7012fa8d51aa8c5, store: [table=TestAcidGuarantees family=A region=76bd866dbccb37465d0501710519b858] 2024-11-07T14:14:49,966 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/87d53f9f3b9d4ab0a7012fa8d51aa8c5 is 175, key is test_row_0/A:col10/1730988889382/Put/seqid=0 2024-11-07T14:14:49,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742048_1224 (size=31255) 2024-11-07T14:14:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T14:14:50,371 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/87d53f9f3b9d4ab0a7012fa8d51aa8c5 2024-11-07T14:14:50,383 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ab328aa0fb744644a1e770a9242e87d6 is 50, key is test_row_0/B:col10/1730988889382/Put/seqid=0 2024-11-07T14:14:50,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742049_1225 (size=12301) 2024-11-07T14:14:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T14:14:50,793 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ab328aa0fb744644a1e770a9242e87d6 2024-11-07T14:14:50,802 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/adf46a8093ad45008308d8929cfddee5 is 50, key is test_row_0/C:col10/1730988889382/Put/seqid=0 2024-11-07T14:14:50,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742050_1226 (size=12301) 2024-11-07T14:14:50,816 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/adf46a8093ad45008308d8929cfddee5 2024-11-07T14:14:50,827 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/A/87d53f9f3b9d4ab0a7012fa8d51aa8c5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/87d53f9f3b9d4ab0a7012fa8d51aa8c5 2024-11-07T14:14:50,835 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/87d53f9f3b9d4ab0a7012fa8d51aa8c5, entries=150, sequenceid=330, filesize=30.5 K 2024-11-07T14:14:50,837 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/B/ab328aa0fb744644a1e770a9242e87d6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ab328aa0fb744644a1e770a9242e87d6 2024-11-07T14:14:50,851 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ab328aa0fb744644a1e770a9242e87d6, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T14:14:50,853 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/.tmp/C/adf46a8093ad45008308d8929cfddee5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/adf46a8093ad45008308d8929cfddee5 2024-11-07T14:14:50,859 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/adf46a8093ad45008308d8929cfddee5, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T14:14:50,860 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 76bd866dbccb37465d0501710519b858 in 1313ms, sequenceid=330, compaction requested=true 2024-11-07T14:14:50,870 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e9f1684ae131459e8c036200b0066355, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/d0c167e562084c869ce5d8029230089a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/fd5813a3696142369d9a4ed093a8f70e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e017b74eea184a608a39ac5255047b3d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/1ec7d5a500094ef28ce0ffdc8790e13b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/6e0c6743b10b48cfafd45a6a43296714, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470199a862344d22960fc5c063c5baa9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a4b971da2c6a495781773d7f82d97bce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/44050868f42040d18e0f46e576719dbe, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/58e158f146964309a89ba47011db7cd2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/147fea8994f44b379e1bd06cc6193812, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/cd60599accdd437ca89e94a49e83f3f9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/b047b6891b394c6188529d295335e73d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/7ccdd843755e4f839904df1786ef126d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/dbbd639f330d4b18acbe7a715e2dac88, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/aea25c51faab4879930a64f8b8ba63fe, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/ed0aa8aeaecd43ac91d25a4000bc6aa5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a0ef704ceb8f40a2a62923ad80ded167, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/0520dc45ca434f62a2e028c7ba3657a2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/c60dbc08ddcc428ebdaa70dfd1c52ce2] to archive 2024-11-07T14:14:50,873 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:14:50,876 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e9f1684ae131459e8c036200b0066355 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e9f1684ae131459e8c036200b0066355 2024-11-07T14:14:50,878 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/d0c167e562084c869ce5d8029230089a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/d0c167e562084c869ce5d8029230089a 2024-11-07T14:14:50,881 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/fd5813a3696142369d9a4ed093a8f70e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/fd5813a3696142369d9a4ed093a8f70e 2024-11-07T14:14:50,883 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e017b74eea184a608a39ac5255047b3d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/e017b74eea184a608a39ac5255047b3d 2024-11-07T14:14:50,885 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/1ec7d5a500094ef28ce0ffdc8790e13b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/1ec7d5a500094ef28ce0ffdc8790e13b 2024-11-07T14:14:50,886 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/6e0c6743b10b48cfafd45a6a43296714 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/6e0c6743b10b48cfafd45a6a43296714 2024-11-07T14:14:50,893 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470199a862344d22960fc5c063c5baa9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470199a862344d22960fc5c063c5baa9 2024-11-07T14:14:50,895 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a4b971da2c6a495781773d7f82d97bce to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a4b971da2c6a495781773d7f82d97bce 2024-11-07T14:14:50,897 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/44050868f42040d18e0f46e576719dbe to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/44050868f42040d18e0f46e576719dbe 2024-11-07T14:14:50,898 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/58e158f146964309a89ba47011db7cd2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/58e158f146964309a89ba47011db7cd2 2024-11-07T14:14:50,899 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/147fea8994f44b379e1bd06cc6193812 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/147fea8994f44b379e1bd06cc6193812 2024-11-07T14:14:50,901 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/cd60599accdd437ca89e94a49e83f3f9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/cd60599accdd437ca89e94a49e83f3f9 2024-11-07T14:14:50,902 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/b047b6891b394c6188529d295335e73d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/b047b6891b394c6188529d295335e73d 2024-11-07T14:14:50,904 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/7ccdd843755e4f839904df1786ef126d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/7ccdd843755e4f839904df1786ef126d 2024-11-07T14:14:50,906 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/dbbd639f330d4b18acbe7a715e2dac88 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/dbbd639f330d4b18acbe7a715e2dac88 2024-11-07T14:14:50,908 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/aea25c51faab4879930a64f8b8ba63fe to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/aea25c51faab4879930a64f8b8ba63fe 2024-11-07T14:14:50,910 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/ed0aa8aeaecd43ac91d25a4000bc6aa5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/ed0aa8aeaecd43ac91d25a4000bc6aa5 2024-11-07T14:14:50,912 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a0ef704ceb8f40a2a62923ad80ded167 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/a0ef704ceb8f40a2a62923ad80ded167 2024-11-07T14:14:50,915 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/0520dc45ca434f62a2e028c7ba3657a2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/0520dc45ca434f62a2e028c7ba3657a2 2024-11-07T14:14:50,919 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/c60dbc08ddcc428ebdaa70dfd1c52ce2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/c60dbc08ddcc428ebdaa70dfd1c52ce2 2024-11-07T14:14:50,927 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/80f2ba3676b248dd8db8cf100968560a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/90f6433380054332b39e18a928ec3105, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/38f2693d4645496aac854c5c6068db1d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9f6faff359b8409089b42e9bab5c51ce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/61baefa55c6f4c048a466f7f9c5eb242, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/35b3cdf037f54d72bbf74654726e8236, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ae6b422c1dec452ea53c369c22b451c7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/c4066e22375d480a80be1178dc985a88, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/5d95e08a92ce4edebe7056d01f41c015, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/af137cfb0fbf447ebd834fffb53820ce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/81b26c1ebac64af49227642f04e85f0f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/2934c9e9a1b14ad6bdabe3f191691f1a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/0b80b94f9bd148d89b74a11e6915ccb6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8412d23a6dc7495fb8dc1758f2ad34a3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/75bda1a208864add8784f2f0676c71a3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/eeace52f66b14c8a872028f69f171b5e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ce7e4305a7b94f69a53c5cd756c48d75, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1ca5cc81255c49d99ea60028937216ad, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8e76150ff44e4b048273510a4fa04d0e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9ceb9147a5a342e793e70f5fe640ab43] to archive 2024-11-07T14:14:50,929 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:14:50,932 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/80f2ba3676b248dd8db8cf100968560a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/80f2ba3676b248dd8db8cf100968560a 2024-11-07T14:14:50,935 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/90f6433380054332b39e18a928ec3105 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/90f6433380054332b39e18a928ec3105 2024-11-07T14:14:50,937 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/38f2693d4645496aac854c5c6068db1d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/38f2693d4645496aac854c5c6068db1d 2024-11-07T14:14:50,941 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9f6faff359b8409089b42e9bab5c51ce to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9f6faff359b8409089b42e9bab5c51ce 2024-11-07T14:14:50,943 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/61baefa55c6f4c048a466f7f9c5eb242 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/61baefa55c6f4c048a466f7f9c5eb242 2024-11-07T14:14:50,945 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/35b3cdf037f54d72bbf74654726e8236 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/35b3cdf037f54d72bbf74654726e8236 2024-11-07T14:14:50,948 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ae6b422c1dec452ea53c369c22b451c7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ae6b422c1dec452ea53c369c22b451c7 2024-11-07T14:14:50,950 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/c4066e22375d480a80be1178dc985a88 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/c4066e22375d480a80be1178dc985a88 2024-11-07T14:14:50,952 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/5d95e08a92ce4edebe7056d01f41c015 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/5d95e08a92ce4edebe7056d01f41c015 2024-11-07T14:14:50,954 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/af137cfb0fbf447ebd834fffb53820ce to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/af137cfb0fbf447ebd834fffb53820ce 2024-11-07T14:14:50,956 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/81b26c1ebac64af49227642f04e85f0f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/81b26c1ebac64af49227642f04e85f0f 2024-11-07T14:14:50,957 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/2934c9e9a1b14ad6bdabe3f191691f1a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/2934c9e9a1b14ad6bdabe3f191691f1a 2024-11-07T14:14:50,959 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/0b80b94f9bd148d89b74a11e6915ccb6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/0b80b94f9bd148d89b74a11e6915ccb6 2024-11-07T14:14:50,961 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8412d23a6dc7495fb8dc1758f2ad34a3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8412d23a6dc7495fb8dc1758f2ad34a3 2024-11-07T14:14:50,963 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/75bda1a208864add8784f2f0676c71a3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/75bda1a208864add8784f2f0676c71a3 2024-11-07T14:14:50,965 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/eeace52f66b14c8a872028f69f171b5e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/eeace52f66b14c8a872028f69f171b5e 2024-11-07T14:14:50,967 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ce7e4305a7b94f69a53c5cd756c48d75 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ce7e4305a7b94f69a53c5cd756c48d75 2024-11-07T14:14:50,974 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1ca5cc81255c49d99ea60028937216ad to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1ca5cc81255c49d99ea60028937216ad 2024-11-07T14:14:50,976 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8e76150ff44e4b048273510a4fa04d0e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/8e76150ff44e4b048273510a4fa04d0e 2024-11-07T14:14:50,979 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9ceb9147a5a342e793e70f5fe640ab43 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/9ceb9147a5a342e793e70f5fe640ab43 2024-11-07T14:14:50,982 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d41b45d79035424eac409197bb94c305, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/df5db178d1674dceb61af5a76aa47a1d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/450f8e3a0516427598e134f21399f556, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c52a82de93864d90a4cbcbdfebe7f8c1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/93c416a35341490891db2ad081dc58b8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/09f6276771f24af2b08069ac00f9702e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/ed35784a053946e0a637068107a3cd84, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d892d45613ae4cf4823b037b311a032f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c68689b662bc43e69a0ffae9b04f5b1f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/19867b3fbe8f409d8dbd490c02d9cc2f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/4bc8a840ba97469499c8c65bb43a3486, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/20b2656e311e4044ac4d77aea87ae626, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f049b2e3a3c0403594d6b3fc318a3c06, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f78fb45b4ade4ebba1347722d2281fb3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f23609498fa94e19a0c9551e429fe63c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/563d0c7449844c18a6d5e70bac88cce7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/2c32c7218974434da3d22342f787147b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/8147d1723bc64c47a2eda605a9253d11, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/b938ab6db5c54c76a4668a13ffdca479, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c00330b392894a7eab204ee6b0d01722] to archive 2024-11-07T14:14:50,984 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:14:50,987 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d41b45d79035424eac409197bb94c305 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d41b45d79035424eac409197bb94c305 2024-11-07T14:14:50,989 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/df5db178d1674dceb61af5a76aa47a1d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/df5db178d1674dceb61af5a76aa47a1d 2024-11-07T14:14:50,991 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/450f8e3a0516427598e134f21399f556 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/450f8e3a0516427598e134f21399f556 2024-11-07T14:14:50,992 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c52a82de93864d90a4cbcbdfebe7f8c1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c52a82de93864d90a4cbcbdfebe7f8c1 2024-11-07T14:14:50,995 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/93c416a35341490891db2ad081dc58b8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/93c416a35341490891db2ad081dc58b8 2024-11-07T14:14:51,007 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/09f6276771f24af2b08069ac00f9702e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/09f6276771f24af2b08069ac00f9702e 2024-11-07T14:14:51,009 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/ed35784a053946e0a637068107a3cd84 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/ed35784a053946e0a637068107a3cd84 2024-11-07T14:14:51,017 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d892d45613ae4cf4823b037b311a032f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d892d45613ae4cf4823b037b311a032f 2024-11-07T14:14:51,020 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c68689b662bc43e69a0ffae9b04f5b1f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c68689b662bc43e69a0ffae9b04f5b1f 2024-11-07T14:14:51,022 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/19867b3fbe8f409d8dbd490c02d9cc2f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/19867b3fbe8f409d8dbd490c02d9cc2f 2024-11-07T14:14:51,024 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/4bc8a840ba97469499c8c65bb43a3486 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/4bc8a840ba97469499c8c65bb43a3486 2024-11-07T14:14:51,025 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/20b2656e311e4044ac4d77aea87ae626 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/20b2656e311e4044ac4d77aea87ae626 2024-11-07T14:14:51,027 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f049b2e3a3c0403594d6b3fc318a3c06 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f049b2e3a3c0403594d6b3fc318a3c06 2024-11-07T14:14:51,029 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f78fb45b4ade4ebba1347722d2281fb3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f78fb45b4ade4ebba1347722d2281fb3 2024-11-07T14:14:51,031 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f23609498fa94e19a0c9551e429fe63c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/f23609498fa94e19a0c9551e429fe63c 2024-11-07T14:14:51,042 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/563d0c7449844c18a6d5e70bac88cce7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/563d0c7449844c18a6d5e70bac88cce7 2024-11-07T14:14:51,045 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/2c32c7218974434da3d22342f787147b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/2c32c7218974434da3d22342f787147b 2024-11-07T14:14:51,046 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/8147d1723bc64c47a2eda605a9253d11 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/8147d1723bc64c47a2eda605a9253d11 2024-11-07T14:14:51,051 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/b938ab6db5c54c76a4668a13ffdca479 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/b938ab6db5c54c76a4668a13ffdca479 2024-11-07T14:14:51,054 DEBUG [StoreCloser-TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c00330b392894a7eab204ee6b0d01722 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/c00330b392894a7eab204ee6b0d01722 2024-11-07T14:14:51,070 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/recovered.edits/333.seqid, newMaxSeqId=333, maxSeqId=4 2024-11-07T14:14:51,072 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. 2024-11-07T14:14:51,072 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 76bd866dbccb37465d0501710519b858: Waiting for close lock at 1730988889547Running coprocessor pre-close hooks at 1730988889547Disabling compacts and flushes for region at 1730988889547Disabling writes for close at 1730988889547Obtaining lock to block concurrent updates at 1730988889547Preparing flush snapshotting stores in 76bd866dbccb37465d0501710519b858 at 1730988889547Finished memstore snapshotting TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1730988889547Flushing stores of TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858. at 1730988889548 (+1 ms)Flushing 76bd866dbccb37465d0501710519b858/A: creating writer at 1730988889548Flushing mob file 76bd866dbccb37465d0501710519b858/A: appending metadata at 1730988889555 (+7 ms)Flushing mob file 76bd866dbccb37465d0501710519b858/A: closing flushed file at 1730988889555Flushing 76bd866dbccb37465d0501710519b858/A: appending metadata at 1730988889966 (+411 ms)Flushing 76bd866dbccb37465d0501710519b858/A: closing flushed file at 1730988889966Flushing 76bd866dbccb37465d0501710519b858/B: creating writer at 1730988890378 (+412 ms)Flushing 76bd866dbccb37465d0501710519b858/B: appending metadata at 1730988890383 (+5 ms)Flushing 76bd866dbccb37465d0501710519b858/B: closing flushed file at 1730988890383Flushing 76bd866dbccb37465d0501710519b858/C: creating writer at 1730988890798 (+415 ms)Flushing 76bd866dbccb37465d0501710519b858/C: appending metadata at 1730988890801 (+3 ms)Flushing 76bd866dbccb37465d0501710519b858/C: closing flushed file at 1730988890801Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@506061ce: reopening flushed file at 1730988890823 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52feb53f: reopening flushed file at 1730988890835 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60052d52: reopening flushed file at 1730988890851 (+16 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 76bd866dbccb37465d0501710519b858 in 1313ms, sequenceid=330, compaction requested=true at 1730988890860 (+9 ms)Writing region close event to WAL at 1730988891059 (+199 ms)Running coprocessor post-close hooks at 1730988891071 (+12 ms)Closed at 1730988891072 (+1 ms) 2024-11-07T14:14:51,074 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,075 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=76bd866dbccb37465d0501710519b858, regionState=CLOSED 2024-11-07T14:14:51,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:14:51,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T14:14:51,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 76bd866dbccb37465d0501710519b858, server=b7cc9d764de6,37737,1730988826985 in 1.6840 sec 2024-11-07T14:14:51,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-07T14:14:51,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76bd866dbccb37465d0501710519b858, UNASSIGN in 1.6910 sec 2024-11-07T14:14:51,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T14:14:51,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6970 sec 2024-11-07T14:14:51,093 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988891093"}]},"ts":"1730988891093"} 2024-11-07T14:14:51,095 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T14:14:51,095 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T14:14:51,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7110 sec 2024-11-07T14:14:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T14:14:51,533 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:51,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T14:14:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:51,535 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T14:14:51,536 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:51,538 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,541 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/recovered.edits] 2024-11-07T14:14:51,543 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470f5c889d1a49ef963c7f87d54bc86b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/470f5c889d1a49ef963c7f87d54bc86b 2024-11-07T14:14:51,544 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/658cca36157446598b1a30568c50030c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/658cca36157446598b1a30568c50030c 2024-11-07T14:14:51,545 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/87d53f9f3b9d4ab0a7012fa8d51aa8c5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/A/87d53f9f3b9d4ab0a7012fa8d51aa8c5 2024-11-07T14:14:51,548 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1db3e87935404b8b928d6bc90c7e00fa to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/1db3e87935404b8b928d6bc90c7e00fa 2024-11-07T14:14:51,549 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/3f3be19c75bf458ab0d32a3ef582a30e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/3f3be19c75bf458ab0d32a3ef582a30e 2024-11-07T14:14:51,550 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ab328aa0fb744644a1e770a9242e87d6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/B/ab328aa0fb744644a1e770a9242e87d6 2024-11-07T14:14:51,552 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/136551f463fe48a6858769306ce3e844 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/136551f463fe48a6858769306ce3e844 2024-11-07T14:14:51,553 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/adf46a8093ad45008308d8929cfddee5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/adf46a8093ad45008308d8929cfddee5 2024-11-07T14:14:51,554 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d1a08dd9143646449426befc5892303a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/C/d1a08dd9143646449426befc5892303a 2024-11-07T14:14:51,557 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/recovered.edits/333.seqid to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858/recovered.edits/333.seqid 2024-11-07T14:14:51,558 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,558 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T14:14:51,558 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T14:14:51,559 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T14:14:51,563 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110736ff491c327e45c98402f208611a9d24_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110736ff491c327e45c98402f208611a9d24_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,564 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110739073c30d16b424ba6fa28ee83898e08_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110739073c30d16b424ba6fa28ee83898e08_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,566 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110740eaa80ef4d04f06a9be4be1235c9548_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110740eaa80ef4d04f06a9be4be1235c9548_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,567 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074389523e90734e33ae40b2956102ae35_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074389523e90734e33ae40b2956102ae35_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,568 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749deb1684476432bb064a0642a716096_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749deb1684476432bb064a0642a716096_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,569 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074cb4fc598e294511ac7f629984fb9437_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074cb4fc598e294511ac7f629984fb9437_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,570 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075fb882c6431245eda00738a0f5c1fc69_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075fb882c6431245eda00738a0f5c1fc69_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,572 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107664ae2f71c8c49e1b944dd337863a11c_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107664ae2f71c8c49e1b944dd337863a11c_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,573 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110771d9234924a443a6b02baa95dfca637f_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110771d9234924a443a6b02baa95dfca637f_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,574 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079d659a2747fe4deb886e885b9e1c5ff9_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079d659a2747fe4deb886e885b9e1c5ff9_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,575 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a03eaaf5e5b347869892a21812eee0ab_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a03eaaf5e5b347869892a21812eee0ab_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,576 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a723985fd9c1469b9484fd9c0729b520_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a723985fd9c1469b9484fd9c0729b520_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,577 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b31a4dc831c344d0b363d02d692900e1_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b31a4dc831c344d0b363d02d692900e1_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,578 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b4884f4b0f144e8197361044d25550ef_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b4884f4b0f144e8197361044d25550ef_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,579 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bbf0d75e8efd4b52895424f2607d5cb0_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bbf0d75e8efd4b52895424f2607d5cb0_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,581 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ea5edb36961e4fd18270993c7bb7bc02_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ea5edb36961e4fd18270993c7bb7bc02_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,582 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ef33edd445a8403485d6d412ad5ae136_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ef33edd445a8403485d6d412ad5ae136_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,583 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fe8ff04fb2844fb99a45703f78bd69ac_76bd866dbccb37465d0501710519b858 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fe8ff04fb2844fb99a45703f78bd69ac_76bd866dbccb37465d0501710519b858 2024-11-07T14:14:51,583 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T14:14:51,586 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:51,588 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T14:14:51,590 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T14:14:51,591 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:51,591 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T14:14:51,592 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730988891591"}]},"ts":"9223372036854775807"} 2024-11-07T14:14:51,594 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T14:14:51,594 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 76bd866dbccb37465d0501710519b858, NAME => 'TestAcidGuarantees,,1730988861420.76bd866dbccb37465d0501710519b858.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T14:14:51,594 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T14:14:51,594 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730988891594"}]},"ts":"9223372036854775807"} 2024-11-07T14:14:51,596 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T14:14:51,597 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:51,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-07T14:14:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T14:14:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T14:14:51,642 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:51,654 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1469413959_22 at /127.0.0.1:57188 [Waiting for operation #361] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1809816429_22 at /127.0.0.1:57164 [Waiting for operation #226] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1809816429_22 at /127.0.0.1:57162 [Waiting for operation #237] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1469413959_22 at /127.0.0.1:51958 [Waiting for operation #326] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=553 (was 634), ProcessCount=11 (was 11), AvailableMemoryMB=1150 (was 246) - AvailableMemoryMB LEAK? - 2024-11-07T14:14:51,664 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=553, ProcessCount=11, AvailableMemoryMB=1136 2024-11-07T14:14:51,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:14:51,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:14:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T14:14:51,668 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:14:51,668 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:51,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-07T14:14:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T14:14:51,669 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:14:51,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742051_1227 (size=959) 2024-11-07T14:14:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T14:14:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T14:14:52,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 2024-11-07T14:14:52,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742052_1228 (size=53) 2024-11-07T14:14:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T14:14:52,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:14:52,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1d800a645cbbc59d7375bdd5295bbe51, disabling compactions & flushes 2024-11-07T14:14:52,485 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:52,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:52,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. after waiting 0 ms 2024-11-07T14:14:52,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:52,485 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:52,485 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d800a645cbbc59d7375bdd5295bbe51: Waiting for close lock at 1730988892485Disabling compacts and flushes for region at 1730988892485Disabling writes for close at 1730988892485Writing region close event to WAL at 1730988892485Closed at 1730988892485 2024-11-07T14:14:52,486 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:14:52,487 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730988892486"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730988892486"}]},"ts":"1730988892486"} 2024-11-07T14:14:52,488 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T14:14:52,489 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:14:52,490 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988892489"}]},"ts":"1730988892489"} 2024-11-07T14:14:52,491 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T14:14:52,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, ASSIGN}] 2024-11-07T14:14:52,493 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, ASSIGN 2024-11-07T14:14:52,493 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, ASSIGN; state=OFFLINE, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=false 2024-11-07T14:14:52,644 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1d800a645cbbc59d7375bdd5295bbe51, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:52,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, ASSIGN because future has completed 2024-11-07T14:14:52,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:14:52,801 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:52,801 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:14:52,802 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,802 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:14:52,802 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,802 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T14:14:52,803 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,805 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d800a645cbbc59d7375bdd5295bbe51 columnFamilyName A 2024-11-07T14:14:52,805 DEBUG [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:52,805 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] regionserver.HStore(327): Store=1d800a645cbbc59d7375bdd5295bbe51/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:52,805 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,807 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d800a645cbbc59d7375bdd5295bbe51 columnFamilyName B 2024-11-07T14:14:52,807 DEBUG [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:52,807 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] regionserver.HStore(327): Store=1d800a645cbbc59d7375bdd5295bbe51/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:52,807 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,809 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d800a645cbbc59d7375bdd5295bbe51 columnFamilyName C 2024-11-07T14:14:52,809 DEBUG [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:14:52,809 INFO [StoreOpener-1d800a645cbbc59d7375bdd5295bbe51-1 {}] regionserver.HStore(327): Store=1d800a645cbbc59d7375bdd5295bbe51/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:14:52,809 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,810 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,810 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,812 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,812 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,812 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:14:52,813 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,815 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:14:52,816 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 1d800a645cbbc59d7375bdd5295bbe51; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74092152, jitterRate=0.10405910015106201}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:14:52,816 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:52,816 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 1d800a645cbbc59d7375bdd5295bbe51: Running coprocessor pre-open hook at 1730988892802Writing region info on filesystem at 1730988892802Initializing all the Stores at 1730988892803 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988892803Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988892803Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988892803Cleaning up temporary data from old regions at 1730988892812 (+9 ms)Running coprocessor post-open hooks at 1730988892816 (+4 ms)Region opened successfully at 1730988892816 2024-11-07T14:14:52,817 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., pid=65, masterSystemTime=1730988892798 2024-11-07T14:14:52,819 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:52,820 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:52,820 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1d800a645cbbc59d7375bdd5295bbe51, regionState=OPEN, openSeqNum=2, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:52,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:14:52,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T14:14:52,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 in 176 msec 2024-11-07T14:14:52,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-07T14:14:52,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, ASSIGN in 333 msec 2024-11-07T14:14:52,828 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:14:52,828 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988892828"}]},"ts":"1730988892828"} 2024-11-07T14:14:52,830 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T14:14:52,830 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:14:52,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-07T14:14:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T14:14:53,813 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:53,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba7ffab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9806cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9229e63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,826 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,833 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T14:14:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b7cc9d764de6,40745,1730988826115 2024-11-07T14:14:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3bffb984 2024-11-07T14:14:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T14:14:53,836 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T14:14:53,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11961dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13dcbe36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,844 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0606ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48345386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,848 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b053e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,850 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2803f9b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ddee8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,859 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afb13fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,861 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1133a8c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5138abf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb437bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,868 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b5aea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,872 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2d7b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,875 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67efa7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2847dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733092ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d658a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289dbcde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,892 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd3bfcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,895 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c10fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@832215d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d135891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26781044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478bceb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,904 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b56ad9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,906 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e95d15d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63eadf7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:14:53,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:14:53,909 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:14:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:14:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:14:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c811576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:14:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:14:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:14:53,911 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:14:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2273e393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:14:53,915 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:53,931 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,931 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:53,932 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,933 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,933 DEBUG [Thread-1079 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,934 DEBUG [Thread-1079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-07T14:14:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:53,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:53,942 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,942 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T14:14:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,944 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:53,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:53,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:53,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:53,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:14:53,954 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:53,967 DEBUG [Thread-1084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:53,986 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,987 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:14:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:14:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:14:53,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:14:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:54,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:14:54,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988954010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988954009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988954012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/732f7ff2475e42519d132f5caf67ae12 is 50, key is test_row_0/A:col10/1730988893929/Put/seqid=0 2024-11-07T14:14:54,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988954019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988954021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T14:14:54,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742053_1229 (size=14341) 2024-11-07T14:14:54,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/732f7ff2475e42519d132f5caf67ae12 2024-11-07T14:14:54,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T14:14:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/08fdbc73454f4c9288750bdd3fdb0a02 is 50, key is test_row_0/B:col10/1730988893929/Put/seqid=0 2024-11-07T14:14:54,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988954132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988954132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988954132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988954142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988954142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742054_1230 (size=12001) 2024-11-07T14:14:54,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T14:14:54,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:54,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T14:14:54,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988954342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988954352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988954352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988954352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988954352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T14:14:54,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:54,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T14:14:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T14:14:54,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/08fdbc73454f4c9288750bdd3fdb0a02 2024-11-07T14:14:54,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/9018ad9ecd474d17a261d274a0517feb is 50, key is test_row_0/C:col10/1730988893929/Put/seqid=0 2024-11-07T14:14:54,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742055_1231 (size=12001) 2024-11-07T14:14:54,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/9018ad9ecd474d17a261d274a0517feb 2024-11-07T14:14:54,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988954652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/732f7ff2475e42519d132f5caf67ae12 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/732f7ff2475e42519d132f5caf67ae12 2024-11-07T14:14:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988954663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988954664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988954664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/732f7ff2475e42519d132f5caf67ae12, entries=200, sequenceid=14, filesize=14.0 K 2024-11-07T14:14:54,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988954664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/08fdbc73454f4c9288750bdd3fdb0a02 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/08fdbc73454f4c9288750bdd3fdb0a02 2024-11-07T14:14:54,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/08fdbc73454f4c9288750bdd3fdb0a02, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T14:14:54,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/9018ad9ecd474d17a261d274a0517feb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9018ad9ecd474d17a261d274a0517feb 2024-11-07T14:14:54,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9018ad9ecd474d17a261d274a0517feb, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T14:14:54,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1d800a645cbbc59d7375bdd5295bbe51 in 750ms, sequenceid=14, compaction requested=false 2024-11-07T14:14:54,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:54,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T14:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:54,721 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T14:14:54,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/f0eb6484398340dcaf363ca68f260d08 is 50, key is test_row_0/A:col10/1730988894010/Put/seqid=0 2024-11-07T14:14:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742056_1232 (size=12001) 2024-11-07T14:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T14:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:55,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988955184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988955185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988955186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988955188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988955189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,228 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/f0eb6484398340dcaf363ca68f260d08 2024-11-07T14:14:55,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/bfb4a33449a04ed691fdcc4d7e3dda8f is 50, key is test_row_0/B:col10/1730988894010/Put/seqid=0 2024-11-07T14:14:55,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742057_1233 (size=12001) 2024-11-07T14:14:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988955292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988955292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988955293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988955302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988955302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988955509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988955509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988955509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988955515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988955515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,664 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/bfb4a33449a04ed691fdcc4d7e3dda8f 2024-11-07T14:14:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/d80c883b5e214e019c7c77431ba5908d is 50, key is test_row_0/C:col10/1730988894010/Put/seqid=0 2024-11-07T14:14:55,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742058_1234 (size=12001) 2024-11-07T14:14:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988955822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988955822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988955822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988955822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988955822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T14:14:56,098 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/d80c883b5e214e019c7c77431ba5908d 2024-11-07T14:14:56,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/f0eb6484398340dcaf363ca68f260d08 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f0eb6484398340dcaf363ca68f260d08 2024-11-07T14:14:56,125 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f0eb6484398340dcaf363ca68f260d08, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T14:14:56,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/bfb4a33449a04ed691fdcc4d7e3dda8f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/bfb4a33449a04ed691fdcc4d7e3dda8f 2024-11-07T14:14:56,136 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/bfb4a33449a04ed691fdcc4d7e3dda8f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T14:14:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/d80c883b5e214e019c7c77431ba5908d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d80c883b5e214e019c7c77431ba5908d 2024-11-07T14:14:56,151 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d80c883b5e214e019c7c77431ba5908d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T14:14:56,153 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1d800a645cbbc59d7375bdd5295bbe51 in 1432ms, sequenceid=37, compaction requested=false 2024-11-07T14:14:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-07T14:14:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-07T14:14:56,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T14:14:56,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2110 sec 2024-11-07T14:14:56,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2350 sec 2024-11-07T14:14:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:56,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T14:14:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:56,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/1a6cbc9f12744dbcbe4c3c2688d3aad7 is 50, key is test_row_0/A:col10/1730988895186/Put/seqid=0 2024-11-07T14:14:56,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988956356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988956359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988956360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988956360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988956361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742059_1235 (size=12001) 2024-11-07T14:14:56,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/1a6cbc9f12744dbcbe4c3c2688d3aad7 2024-11-07T14:14:56,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/71a4d7a5326f4232a64fbf3914b44fd2 is 50, key is test_row_0/B:col10/1730988895186/Put/seqid=0 2024-11-07T14:14:56,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742060_1236 (size=12001) 2024-11-07T14:14:56,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988956472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988956473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988956473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988956473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988956473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,639 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:14:56,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988956682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988956682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988956682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988956682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988956683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:56,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/71a4d7a5326f4232a64fbf3914b44fd2 2024-11-07T14:14:56,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/2ae168f354c146d998f7d247a8d37b47 is 50, key is test_row_0/C:col10/1730988895186/Put/seqid=0 2024-11-07T14:14:56,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742061_1237 (size=12001) 2024-11-07T14:14:56,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/2ae168f354c146d998f7d247a8d37b47 2024-11-07T14:14:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/1a6cbc9f12744dbcbe4c3c2688d3aad7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/1a6cbc9f12744dbcbe4c3c2688d3aad7 2024-11-07T14:14:56,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/1a6cbc9f12744dbcbe4c3c2688d3aad7, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T14:14:56,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/71a4d7a5326f4232a64fbf3914b44fd2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/71a4d7a5326f4232a64fbf3914b44fd2 2024-11-07T14:14:56,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/71a4d7a5326f4232a64fbf3914b44fd2, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T14:14:56,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/2ae168f354c146d998f7d247a8d37b47 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2ae168f354c146d998f7d247a8d37b47 2024-11-07T14:14:56,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2ae168f354c146d998f7d247a8d37b47, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T14:14:56,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 1d800a645cbbc59d7375bdd5295bbe51 in 594ms, sequenceid=52, compaction requested=true 2024-11-07T14:14:56,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:56,931 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:56,931 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:56,932 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:56,932 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:14:56,932 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:56,932 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/732f7ff2475e42519d132f5caf67ae12, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f0eb6484398340dcaf363ca68f260d08, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/1a6cbc9f12744dbcbe4c3c2688d3aad7] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=37.4 K 2024-11-07T14:14:56,933 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:56,933 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:14:56,933 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:56,933 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/08fdbc73454f4c9288750bdd3fdb0a02, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/bfb4a33449a04ed691fdcc4d7e3dda8f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/71a4d7a5326f4232a64fbf3914b44fd2] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=35.2 K 2024-11-07T14:14:56,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732f7ff2475e42519d132f5caf67ae12, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988893920 2024-11-07T14:14:56,934 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 08fdbc73454f4c9288750bdd3fdb0a02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988893929 2024-11-07T14:14:56,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0eb6484398340dcaf363ca68f260d08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730988894010 2024-11-07T14:14:56,934 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb4a33449a04ed691fdcc4d7e3dda8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730988894010 2024-11-07T14:14:56,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a6cbc9f12744dbcbe4c3c2688d3aad7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730988895186 2024-11-07T14:14:56,935 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a4d7a5326f4232a64fbf3914b44fd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730988895186 2024-11-07T14:14:56,967 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:56,968 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/4ee04698df9d43a6bfb4d4850dfbb6a2 is 50, key is test_row_0/A:col10/1730988895186/Put/seqid=0 2024-11-07T14:14:56,978 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:56,978 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/ba14eece9cab4046bf02adb556109deb is 50, key is test_row_0/B:col10/1730988895186/Put/seqid=0 2024-11-07T14:14:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:56,997 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:56,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:14:57,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742063_1239 (size=12104) 2024-11-07T14:14:57,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988957011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988957011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/61d165453f4e41d1af78233d08ff6f6b is 50, key is test_row_0/A:col10/1730988896352/Put/seqid=0 2024-11-07T14:14:57,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988957017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988957017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988957019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,032 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/ba14eece9cab4046bf02adb556109deb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/ba14eece9cab4046bf02adb556109deb 2024-11-07T14:14:57,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742062_1238 (size=12104) 2024-11-07T14:14:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,042 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into ba14eece9cab4046bf02adb556109deb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:57,042 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:57,042 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988896931; duration=0sec 2024-11-07T14:14:57,042 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:57,042 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:14:57,042 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:57,044 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:57,044 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:14:57,044 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:57,044 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9018ad9ecd474d17a261d274a0517feb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d80c883b5e214e019c7c77431ba5908d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2ae168f354c146d998f7d247a8d37b47] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=35.2 K 2024-11-07T14:14:57,045 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9018ad9ecd474d17a261d274a0517feb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988893929 2024-11-07T14:14:57,046 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d80c883b5e214e019c7c77431ba5908d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730988894010 2024-11-07T14:14:57,049 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/4ee04698df9d43a6bfb4d4850dfbb6a2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4ee04698df9d43a6bfb4d4850dfbb6a2 2024-11-07T14:14:57,049 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae168f354c146d998f7d247a8d37b47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730988895186 2024-11-07T14:14:57,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742064_1240 (size=12001) 2024-11-07T14:14:57,059 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into 4ee04698df9d43a6bfb4d4850dfbb6a2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:57,059 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:57,059 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988896930; duration=0sec 2024-11-07T14:14:57,059 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:57,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/61d165453f4e41d1af78233d08ff6f6b 2024-11-07T14:14:57,059 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:14:57,069 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#200 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:57,069 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f3999e84104c4c8fa48e79153004b13f is 50, key is test_row_0/C:col10/1730988895186/Put/seqid=0 2024-11-07T14:14:57,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/f7cf9ab8004f4be8be5f0b1828d83087 is 50, key is test_row_0/B:col10/1730988896352/Put/seqid=0 2024-11-07T14:14:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742066_1242 (size=12001) 2024-11-07T14:14:57,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742065_1241 (size=12104) 2024-11-07T14:14:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988957132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988957133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988957133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988957143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988957143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988957342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988957342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988957342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988957352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988957352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/f7cf9ab8004f4be8be5f0b1828d83087 2024-11-07T14:14:57,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4db46817319b4f118ae32f0e81f8bc32 is 50, key is test_row_0/C:col10/1730988896352/Put/seqid=0 2024-11-07T14:14:57,533 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f3999e84104c4c8fa48e79153004b13f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f3999e84104c4c8fa48e79153004b13f 2024-11-07T14:14:57,542 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into f3999e84104c4c8fa48e79153004b13f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:57,542 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:57,542 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=13, startTime=1730988896931; duration=0sec 2024-11-07T14:14:57,542 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:57,542 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:14:57,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742067_1243 (size=12001) 2024-11-07T14:14:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988957652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988957652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988957653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988957662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988957662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:57,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4db46817319b4f118ae32f0e81f8bc32 2024-11-07T14:14:57,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/61d165453f4e41d1af78233d08ff6f6b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/61d165453f4e41d1af78233d08ff6f6b 2024-11-07T14:14:57,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/61d165453f4e41d1af78233d08ff6f6b, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T14:14:57,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/f7cf9ab8004f4be8be5f0b1828d83087 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/f7cf9ab8004f4be8be5f0b1828d83087 2024-11-07T14:14:57,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/f7cf9ab8004f4be8be5f0b1828d83087, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T14:14:57,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4db46817319b4f118ae32f0e81f8bc32 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4db46817319b4f118ae32f0e81f8bc32 2024-11-07T14:14:57,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4db46817319b4f118ae32f0e81f8bc32, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T14:14:57,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1d800a645cbbc59d7375bdd5295bbe51 in 976ms, sequenceid=79, compaction requested=false 2024-11-07T14:14:57,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T14:14:58,103 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:58,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-07T14:14:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T14:14:58,104 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:58,105 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:58,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:58,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:58,164 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:58,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:14:58,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/fd4a86f614e44bb185d870632a9919ec is 50, key is test_row_0/A:col10/1730988897015/Put/seqid=0 2024-11-07T14:14:58,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742068_1244 (size=14337) 2024-11-07T14:14:58,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/fd4a86f614e44bb185d870632a9919ec 2024-11-07T14:14:58,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/e440785361f14fb5b390f37ea16c6a99 is 50, key is test_row_0/B:col10/1730988897015/Put/seqid=0 2024-11-07T14:14:58,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742069_1245 (size=9657) 2024-11-07T14:14:58,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/e440785361f14fb5b390f37ea16c6a99 2024-11-07T14:14:58,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/47622c4b84f8469988d508b62a96909e is 50, key is test_row_0/C:col10/1730988897015/Put/seqid=0 2024-11-07T14:14:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988958194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988958194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988958196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988958197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988958197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742070_1246 (size=9657) 2024-11-07T14:14:58,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/47622c4b84f8469988d508b62a96909e 2024-11-07T14:14:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T14:14:58,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/fd4a86f614e44bb185d870632a9919ec as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/fd4a86f614e44bb185d870632a9919ec 2024-11-07T14:14:58,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/fd4a86f614e44bb185d870632a9919ec, entries=200, sequenceid=93, filesize=14.0 K 2024-11-07T14:14:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/e440785361f14fb5b390f37ea16c6a99 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e440785361f14fb5b390f37ea16c6a99 2024-11-07T14:14:58,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e440785361f14fb5b390f37ea16c6a99, entries=100, sequenceid=93, filesize=9.4 K 2024-11-07T14:14:58,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/47622c4b84f8469988d508b62a96909e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/47622c4b84f8469988d508b62a96909e 2024-11-07T14:14:58,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/47622c4b84f8469988d508b62a96909e, entries=100, sequenceid=93, filesize=9.4 K 2024-11-07T14:14:58,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1d800a645cbbc59d7375bdd5295bbe51 in 71ms, sequenceid=93, compaction requested=true 2024-11-07T14:14:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:58,236 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:58,236 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:58,237 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:58,237 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:14:58,237 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:58,237 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4ee04698df9d43a6bfb4d4850dfbb6a2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/61d165453f4e41d1af78233d08ff6f6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/fd4a86f614e44bb185d870632a9919ec] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=37.5 K 2024-11-07T14:14:58,239 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:58,239 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:14:58,239 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:58,239 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/ba14eece9cab4046bf02adb556109deb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/f7cf9ab8004f4be8be5f0b1828d83087, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e440785361f14fb5b390f37ea16c6a99] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=33.0 K 2024-11-07T14:14:58,239 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee04698df9d43a6bfb4d4850dfbb6a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730988895186 2024-11-07T14:14:58,240 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting ba14eece9cab4046bf02adb556109deb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730988895186 2024-11-07T14:14:58,240 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61d165453f4e41d1af78233d08ff6f6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730988896352 2024-11-07T14:14:58,241 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f7cf9ab8004f4be8be5f0b1828d83087, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730988896352 2024-11-07T14:14:58,241 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd4a86f614e44bb185d870632a9919ec, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730988897015 2024-11-07T14:14:58,242 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e440785361f14fb5b390f37ea16c6a99, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730988897015 2024-11-07T14:14:58,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T14:14:58,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:58,260 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:14:58,268 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:58,268 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/2ab499e05a774537bb68b34ad2a84aec is 50, key is test_row_0/A:col10/1730988897015/Put/seqid=0 2024-11-07T14:14:58,272 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:58,273 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/20bc14f2616146eab50d670afdc30def is 50, key is test_row_0/B:col10/1730988897015/Put/seqid=0 2024-11-07T14:14:58,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/784f60d52ffc4dc78281072c23e27b37 is 50, key is test_row_0/A:col10/1730988898195/Put/seqid=0 2024-11-07T14:14:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:58,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988958315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988958317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988958319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988958319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988958319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742072_1248 (size=12207) 2024-11-07T14:14:58,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742071_1247 (size=12207) 2024-11-07T14:14:58,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742073_1249 (size=12001) 2024-11-07T14:14:58,356 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/784f60d52ffc4dc78281072c23e27b37 2024-11-07T14:14:58,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/9137498bbcaa4cf6a0279b7ccd4af88e is 50, key is test_row_0/B:col10/1730988898195/Put/seqid=0 2024-11-07T14:14:58,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742074_1250 (size=12001) 2024-11-07T14:14:58,385 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/9137498bbcaa4cf6a0279b7ccd4af88e 2024-11-07T14:14:58,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c8cdcfa97284438b98d95cd10226d85f is 50, key is test_row_0/C:col10/1730988898195/Put/seqid=0 2024-11-07T14:14:58,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742075_1251 (size=12001) 2024-11-07T14:14:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T14:14:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988958432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988958432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988958432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988958432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988958432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988958642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988958642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988958642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988958642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988958642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T14:14:58,740 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/20bc14f2616146eab50d670afdc30def as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/20bc14f2616146eab50d670afdc30def 2024-11-07T14:14:58,745 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into 20bc14f2616146eab50d670afdc30def(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:58,745 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:58,745 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988898236; duration=0sec 2024-11-07T14:14:58,745 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:58,745 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:14:58,746 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:58,746 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:58,747 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:14:58,747 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:58,747 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f3999e84104c4c8fa48e79153004b13f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4db46817319b4f118ae32f0e81f8bc32, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/47622c4b84f8469988d508b62a96909e] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=33.0 K 2024-11-07T14:14:58,747 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f3999e84104c4c8fa48e79153004b13f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730988895186 2024-11-07T14:14:58,747 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4db46817319b4f118ae32f0e81f8bc32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730988896352 2024-11-07T14:14:58,748 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 47622c4b84f8469988d508b62a96909e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730988897015 2024-11-07T14:14:58,755 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:58,756 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/e308f0874da34cc1bbc5eaffaf8f8767 is 50, key is test_row_0/C:col10/1730988897015/Put/seqid=0 2024-11-07T14:14:58,767 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/2ab499e05a774537bb68b34ad2a84aec as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2ab499e05a774537bb68b34ad2a84aec 2024-11-07T14:14:58,778 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into 2ab499e05a774537bb68b34ad2a84aec(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:58,778 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:58,778 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988898236; duration=0sec 2024-11-07T14:14:58,778 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:58,778 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:14:58,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742076_1252 (size=12207) 2024-11-07T14:14:58,801 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/e308f0874da34cc1bbc5eaffaf8f8767 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e308f0874da34cc1bbc5eaffaf8f8767 2024-11-07T14:14:58,807 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into e308f0874da34cc1bbc5eaffaf8f8767(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:58,807 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:58,807 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=13, startTime=1730988898236; duration=0sec 2024-11-07T14:14:58,807 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:58,807 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:14:58,821 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c8cdcfa97284438b98d95cd10226d85f 2024-11-07T14:14:58,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/784f60d52ffc4dc78281072c23e27b37 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/784f60d52ffc4dc78281072c23e27b37 2024-11-07T14:14:58,842 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/784f60d52ffc4dc78281072c23e27b37, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T14:14:58,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/9137498bbcaa4cf6a0279b7ccd4af88e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9137498bbcaa4cf6a0279b7ccd4af88e 2024-11-07T14:14:58,853 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9137498bbcaa4cf6a0279b7ccd4af88e, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T14:14:58,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c8cdcfa97284438b98d95cd10226d85f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8cdcfa97284438b98d95cd10226d85f 2024-11-07T14:14:58,884 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8cdcfa97284438b98d95cd10226d85f, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T14:14:58,887 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1d800a645cbbc59d7375bdd5295bbe51 in 627ms, sequenceid=118, compaction requested=false 2024-11-07T14:14:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-07T14:14:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-07T14:14:58,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-07T14:14:58,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 784 msec 2024-11-07T14:14:58,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 790 msec 2024-11-07T14:14:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:58,959 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:58,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T14:14:58,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/8662e79f6d914ef4b140fa4f4c6eedba is 50, key is test_row_0/A:col10/1730988898958/Put/seqid=0 2024-11-07T14:14:59,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988958993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988958995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988958995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988958997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988958997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742077_1253 (size=12101) 2024-11-07T14:14:59,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/8662e79f6d914ef4b140fa4f4c6eedba 2024-11-07T14:14:59,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/5c7bda094443439d86d131aca449cf20 is 50, key is test_row_0/B:col10/1730988898958/Put/seqid=0 2024-11-07T14:14:59,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742078_1254 (size=12101) 2024-11-07T14:14:59,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/5c7bda094443439d86d131aca449cf20 2024-11-07T14:14:59,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988959112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988959112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988959112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988959112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988959112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4c48ac5bf3414103bd35aca06d592606 is 50, key is test_row_0/C:col10/1730988898958/Put/seqid=0 2024-11-07T14:14:59,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742079_1255 (size=12101) 2024-11-07T14:14:59,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4c48ac5bf3414103bd35aca06d592606 2024-11-07T14:14:59,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/8662e79f6d914ef4b140fa4f4c6eedba as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8662e79f6d914ef4b140fa4f4c6eedba 2024-11-07T14:14:59,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8662e79f6d914ef4b140fa4f4c6eedba, entries=150, sequenceid=137, filesize=11.8 K 2024-11-07T14:14:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/5c7bda094443439d86d131aca449cf20 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/5c7bda094443439d86d131aca449cf20 2024-11-07T14:14:59,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/5c7bda094443439d86d131aca449cf20, entries=150, sequenceid=137, filesize=11.8 K 2024-11-07T14:14:59,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4c48ac5bf3414103bd35aca06d592606 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4c48ac5bf3414103bd35aca06d592606 2024-11-07T14:14:59,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4c48ac5bf3414103bd35aca06d592606, entries=150, sequenceid=137, filesize=11.8 K 2024-11-07T14:14:59,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=127.47 KB/130530 for 1d800a645cbbc59d7375bdd5295bbe51 in 259ms, sequenceid=137, compaction requested=true 2024-11-07T14:14:59,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:59,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:14:59,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:59,219 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:59,219 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:59,221 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:59,221 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:14:59,221 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,221 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2ab499e05a774537bb68b34ad2a84aec, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/784f60d52ffc4dc78281072c23e27b37, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8662e79f6d914ef4b140fa4f4c6eedba] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=35.5 K 2024-11-07T14:14:59,221 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:59,221 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:14:59,221 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,221 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/20bc14f2616146eab50d670afdc30def, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9137498bbcaa4cf6a0279b7ccd4af88e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/5c7bda094443439d86d131aca449cf20] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=35.5 K 2024-11-07T14:14:59,222 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 20bc14f2616146eab50d670afdc30def, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730988896997 2024-11-07T14:14:59,222 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab499e05a774537bb68b34ad2a84aec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730988896997 2024-11-07T14:14:59,223 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9137498bbcaa4cf6a0279b7ccd4af88e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730988898192 2024-11-07T14:14:59,223 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 784f60d52ffc4dc78281072c23e27b37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730988898192 2024-11-07T14:14:59,223 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c7bda094443439d86d131aca449cf20, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730988898303 2024-11-07T14:14:59,223 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8662e79f6d914ef4b140fa4f4c6eedba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730988898303 2024-11-07T14:14:59,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:14:59,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:59,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:14:59,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:59,241 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T14:14:59,242 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/aa6e27e6273a4e54be10fbb0fa3d741c is 50, key is test_row_0/A:col10/1730988898958/Put/seqid=0 2024-11-07T14:14:59,242 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:14:59,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:14:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-07T14:14:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T14:14:59,246 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:14:59,247 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:14:59,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:14:59,252 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:59,253 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/8fb12b0c1b984f0e976ed5e6aa2b2cf2 is 50, key is test_row_0/B:col10/1730988898958/Put/seqid=0 2024-11-07T14:14:59,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742081_1257 (size=12409) 2024-11-07T14:14:59,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742080_1256 (size=12409) 2024-11-07T14:14:59,306 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/aa6e27e6273a4e54be10fbb0fa3d741c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/aa6e27e6273a4e54be10fbb0fa3d741c 2024-11-07T14:14:59,314 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into aa6e27e6273a4e54be10fbb0fa3d741c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:59,314 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:59,314 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988899219; duration=0sec 2024-11-07T14:14:59,315 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:14:59,315 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:14:59,315 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:14:59,316 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:14:59,316 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:14:59,316 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,317 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e308f0874da34cc1bbc5eaffaf8f8767, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8cdcfa97284438b98d95cd10226d85f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4c48ac5bf3414103bd35aca06d592606] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=35.5 K 2024-11-07T14:14:59,319 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e308f0874da34cc1bbc5eaffaf8f8767, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730988896997 2024-11-07T14:14:59,320 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8cdcfa97284438b98d95cd10226d85f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730988898192 2024-11-07T14:14:59,320 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c48ac5bf3414103bd35aca06d592606, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730988898303 2024-11-07T14:14:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:14:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:14:59,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T14:14:59,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/332af80fa04e4633804e88a58fe28b6b is 50, key is test_row_0/A:col10/1730988899323/Put/seqid=0 2024-11-07T14:14:59,341 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:14:59,342 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/26b250f8523a4c1f87b26ac0530ed63a is 50, key is test_row_0/C:col10/1730988898958/Put/seqid=0 2024-11-07T14:14:59,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988959335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988959336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988959340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988959340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T14:14:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988959346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742082_1258 (size=12151) 2024-11-07T14:14:59,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742083_1259 (size=12409) 2024-11-07T14:14:59,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T14:14:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,412 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/26b250f8523a4c1f87b26ac0530ed63a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/26b250f8523a4c1f87b26ac0530ed63a 2024-11-07T14:14:59,419 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into 26b250f8523a4c1f87b26ac0530ed63a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:59,419 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:59,419 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=13, startTime=1730988899234; duration=0sec 2024-11-07T14:14:59,419 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:59,419 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:14:59,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988959452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988959452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988959462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988959462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988959462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T14:14:59,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:59,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T14:14:59,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988959662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988959666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988959673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988959674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988959674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,695 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/8fb12b0c1b984f0e976ed5e6aa2b2cf2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8fb12b0c1b984f0e976ed5e6aa2b2cf2 2024-11-07T14:14:59,701 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into 8fb12b0c1b984f0e976ed5e6aa2b2cf2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:14:59,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:14:59,701 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988899219; duration=0sec 2024-11-07T14:14:59,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:14:59,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:14:59,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T14:14:59,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:59,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/332af80fa04e4633804e88a58fe28b6b 2024-11-07T14:14:59,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/d2197c08dcac4ea9b27451f704be8656 is 50, key is test_row_0/B:col10/1730988899323/Put/seqid=0 2024-11-07T14:14:59,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742084_1260 (size=12151) 2024-11-07T14:14:59,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/d2197c08dcac4ea9b27451f704be8656 2024-11-07T14:14:59,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f0a46c1183324eb68780a81be00ca770 is 50, key is test_row_0/C:col10/1730988899323/Put/seqid=0 2024-11-07T14:14:59,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742085_1261 (size=12151) 2024-11-07T14:14:59,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T14:14:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:14:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:14:59,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:14:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T14:14:59,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988959972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988959983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988959983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988959983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:14:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988959983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:14:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:14:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T14:15:00,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:00,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:00,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:00,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:00,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:00,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T14:15:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:00,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:00,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:00,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:00,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f0a46c1183324eb68780a81be00ca770 2024-11-07T14:15:00,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/332af80fa04e4633804e88a58fe28b6b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/332af80fa04e4633804e88a58fe28b6b 2024-11-07T14:15:00,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/332af80fa04e4633804e88a58fe28b6b, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T14:15:00,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/d2197c08dcac4ea9b27451f704be8656 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/d2197c08dcac4ea9b27451f704be8656 2024-11-07T14:15:00,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/d2197c08dcac4ea9b27451f704be8656, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T14:15:00,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f0a46c1183324eb68780a81be00ca770 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f0a46c1183324eb68780a81be00ca770 2024-11-07T14:15:00,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f0a46c1183324eb68780a81be00ca770, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T14:15:00,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1d800a645cbbc59d7375bdd5295bbe51 in 952ms, sequenceid=161, compaction requested=false 2024-11-07T14:15:00,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:00,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T14:15:00,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:00,325 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:15:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/e1514b23cca74281b46e3d712e3a40d2 is 50, key is test_row_0/A:col10/1730988899332/Put/seqid=0 2024-11-07T14:15:00,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742086_1262 (size=9757) 2024-11-07T14:15:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T14:15:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:00,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:00,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988960505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988960507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988960509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988960512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988960512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988960623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988960623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988960623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988960623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988960623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,738 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/e1514b23cca74281b46e3d712e3a40d2 2024-11-07T14:15:00,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/63d5c244c4944b7d8021611a71d417b3 is 50, key is test_row_0/B:col10/1730988899332/Put/seqid=0 2024-11-07T14:15:00,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742087_1263 (size=9757) 2024-11-07T14:15:00,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988960832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988960832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988960833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988960833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988960833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988961142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988961143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988961143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988961143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988961143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,193 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/63d5c244c4944b7d8021611a71d417b3 2024-11-07T14:15:01,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4f9ca0aaf3994b3ba4f10462ead14109 is 50, key is test_row_0/C:col10/1730988899332/Put/seqid=0 2024-11-07T14:15:01,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742088_1264 (size=9757) 2024-11-07T14:15:01,218 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4f9ca0aaf3994b3ba4f10462ead14109 2024-11-07T14:15:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/e1514b23cca74281b46e3d712e3a40d2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e1514b23cca74281b46e3d712e3a40d2 2024-11-07T14:15:01,237 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e1514b23cca74281b46e3d712e3a40d2, entries=100, sequenceid=176, filesize=9.5 K 2024-11-07T14:15:01,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/63d5c244c4944b7d8021611a71d417b3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63d5c244c4944b7d8021611a71d417b3 2024-11-07T14:15:01,246 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63d5c244c4944b7d8021611a71d417b3, entries=100, sequenceid=176, filesize=9.5 K 2024-11-07T14:15:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4f9ca0aaf3994b3ba4f10462ead14109 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f9ca0aaf3994b3ba4f10462ead14109 2024-11-07T14:15:01,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f9ca0aaf3994b3ba4f10462ead14109, entries=100, sequenceid=176, filesize=9.5 K 2024-11-07T14:15:01,254 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1d800a645cbbc59d7375bdd5295bbe51 in 929ms, sequenceid=176, compaction requested=true 2024-11-07T14:15:01,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:01,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:01,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-07T14:15:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-07T14:15:01,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T14:15:01,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0100 sec 2024-11-07T14:15:01,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0180 sec 2024-11-07T14:15:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T14:15:01,393 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:01,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-07T14:15:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T14:15:01,394 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:01,395 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:01,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T14:15:01,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T14:15:01,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:01,553 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:15:01,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/8fda8b072ed54087876043c8b3ce905d is 50, key is test_row_0/A:col10/1730988900506/Put/seqid=0 2024-11-07T14:15:01,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742089_1265 (size=12151) 2024-11-07T14:15:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988961668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988961669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988961670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988961672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988961680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T14:15:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988961782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988961782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988961783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988961792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988961802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,962 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/8fda8b072ed54087876043c8b3ce905d 2024-11-07T14:15:01,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/b7d32a682af14c95976fbc9857d10c35 is 50, key is test_row_0/B:col10/1730988900506/Put/seqid=0 2024-11-07T14:15:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988961992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988961992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:01,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988961993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988962002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988962013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742090_1266 (size=12151) 2024-11-07T14:15:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T14:15:02,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988962302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988962302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988962302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988962312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988962322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,420 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/b7d32a682af14c95976fbc9857d10c35 2024-11-07T14:15:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f321ee3facbd4d2faf442332f92a758f is 50, key is test_row_0/C:col10/1730988900506/Put/seqid=0 2024-11-07T14:15:02,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742091_1267 (size=12151) 2024-11-07T14:15:02,443 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f321ee3facbd4d2faf442332f92a758f 2024-11-07T14:15:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/8fda8b072ed54087876043c8b3ce905d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8fda8b072ed54087876043c8b3ce905d 2024-11-07T14:15:02,457 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8fda8b072ed54087876043c8b3ce905d, entries=150, sequenceid=200, filesize=11.9 K 2024-11-07T14:15:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/b7d32a682af14c95976fbc9857d10c35 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/b7d32a682af14c95976fbc9857d10c35 2024-11-07T14:15:02,467 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/b7d32a682af14c95976fbc9857d10c35, entries=150, sequenceid=200, filesize=11.9 K 2024-11-07T14:15:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f321ee3facbd4d2faf442332f92a758f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f321ee3facbd4d2faf442332f92a758f 2024-11-07T14:15:02,472 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f321ee3facbd4d2faf442332f92a758f, entries=150, sequenceid=200, filesize=11.9 K 2024-11-07T14:15:02,473 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1d800a645cbbc59d7375bdd5295bbe51 in 920ms, sequenceid=200, compaction requested=true 2024-11-07T14:15:02,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:02,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:02,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-07T14:15:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-07T14:15:02,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T14:15:02,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0840 sec 2024-11-07T14:15:02,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0900 sec 2024-11-07T14:15:02,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T14:15:02,543 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:02,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-07T14:15:02,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T14:15:02,545 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:02,547 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:02,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T14:15:02,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T14:15:02,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:02,702 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:15:02,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/713f410b216c46f1a66f23bad7813e32 is 50, key is test_row_0/A:col10/1730988901654/Put/seqid=0 2024-11-07T14:15:02,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742092_1268 (size=12151) 2024-11-07T14:15:02,723 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/713f410b216c46f1a66f23bad7813e32 2024-11-07T14:15:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/fb688aaee4ed4d9a867513420694862d is 50, key is test_row_0/B:col10/1730988901654/Put/seqid=0 2024-11-07T14:15:02,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742093_1269 (size=12151) 2024-11-07T14:15:02,743 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/fb688aaee4ed4d9a867513420694862d 2024-11-07T14:15:02,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c8bd56f024f1446198fce6b8fb39efa2 is 50, key is test_row_0/C:col10/1730988901654/Put/seqid=0 2024-11-07T14:15:02,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742094_1270 (size=12151) 2024-11-07T14:15:02,761 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c8bd56f024f1446198fce6b8fb39efa2 2024-11-07T14:15:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/713f410b216c46f1a66f23bad7813e32 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/713f410b216c46f1a66f23bad7813e32 2024-11-07T14:15:02,771 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/713f410b216c46f1a66f23bad7813e32, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T14:15:02,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/fb688aaee4ed4d9a867513420694862d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/fb688aaee4ed4d9a867513420694862d 2024-11-07T14:15:02,777 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/fb688aaee4ed4d9a867513420694862d, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T14:15:02,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c8bd56f024f1446198fce6b8fb39efa2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8bd56f024f1446198fce6b8fb39efa2 2024-11-07T14:15:02,807 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8bd56f024f1446198fce6b8fb39efa2, entries=150, sequenceid=212, filesize=11.9 K 2024-11-07T14:15:02,812 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 1d800a645cbbc59d7375bdd5295bbe51 in 109ms, sequenceid=212, compaction requested=true 2024-11-07T14:15:02,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:02,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:02,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-07T14:15:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-07T14:15:02,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-07T14:15:02,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 268 msec 2024-11-07T14:15:02,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 275 msec 2024-11-07T14:15:02,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:02,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:15:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:02,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/2f069f50beee42b4b7479452d89e0b3e is 50, key is test_row_0/A:col10/1730988902819/Put/seqid=0 2024-11-07T14:15:02,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742095_1271 (size=14541) 2024-11-07T14:15:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T14:15:02,863 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:02,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-07T14:15:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T14:15:02,869 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:02,872 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:02,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:02,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988962879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988962881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988962883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988962883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988962884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T14:15:02,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988962992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988962993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988963002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988963003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988963003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T14:15:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T14:15:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T14:15:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988963202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988963202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988963212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988963213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988963213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/2f069f50beee42b4b7479452d89e0b3e 2024-11-07T14:15:03,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/943e615800f247fc9ef7231011ff2d54 is 50, key is test_row_0/B:col10/1730988902819/Put/seqid=0 2024-11-07T14:15:03,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742096_1272 (size=12151) 2024-11-07T14:15:03,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T14:15:03,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:03,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T14:15:03,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:03,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T14:15:03,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988963512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988963512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988963522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988963523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988963523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:03,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:03,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:03,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T14:15:03,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:03,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/943e615800f247fc9ef7231011ff2d54 2024-11-07T14:15:03,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/d66b1b6afff1438eb2ce375ff81b9289 is 50, key is test_row_0/C:col10/1730988902819/Put/seqid=0 2024-11-07T14:15:03,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742097_1273 (size=12151) 2024-11-07T14:15:03,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/d66b1b6afff1438eb2ce375ff81b9289 2024-11-07T14:15:03,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/2f069f50beee42b4b7479452d89e0b3e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2f069f50beee42b4b7479452d89e0b3e 2024-11-07T14:15:03,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2f069f50beee42b4b7479452d89e0b3e, entries=200, sequenceid=224, filesize=14.2 K 2024-11-07T14:15:03,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/943e615800f247fc9ef7231011ff2d54 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/943e615800f247fc9ef7231011ff2d54 2024-11-07T14:15:03,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/943e615800f247fc9ef7231011ff2d54, entries=150, sequenceid=224, filesize=11.9 K 2024-11-07T14:15:03,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/d66b1b6afff1438eb2ce375ff81b9289 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d66b1b6afff1438eb2ce375ff81b9289 2024-11-07T14:15:03,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d66b1b6afff1438eb2ce375ff81b9289, entries=150, sequenceid=224, filesize=11.9 K 2024-11-07T14:15:03,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1d800a645cbbc59d7375bdd5295bbe51 in 899ms, sequenceid=224, compaction requested=true 2024-11-07T14:15:03,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:03,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:03,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:03,732 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-07T14:15:03,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:03,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:03,732 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-07T14:15:03,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:03,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:03,734 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 70770 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-07T14:15:03,734 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:15:03,734 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,734 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8fb12b0c1b984f0e976ed5e6aa2b2cf2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/d2197c08dcac4ea9b27451f704be8656, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63d5c244c4944b7d8021611a71d417b3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/b7d32a682af14c95976fbc9857d10c35, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/fb688aaee4ed4d9a867513420694862d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/943e615800f247fc9ef7231011ff2d54] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=69.1 K 2024-11-07T14:15:03,734 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73160 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-07T14:15:03,734 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:15:03,734 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb12b0c1b984f0e976ed5e6aa2b2cf2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730988898303 2024-11-07T14:15:03,734 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,734 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/aa6e27e6273a4e54be10fbb0fa3d741c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/332af80fa04e4633804e88a58fe28b6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e1514b23cca74281b46e3d712e3a40d2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8fda8b072ed54087876043c8b3ce905d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/713f410b216c46f1a66f23bad7813e32, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2f069f50beee42b4b7479452d89e0b3e] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=71.4 K 2024-11-07T14:15:03,735 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d2197c08dcac4ea9b27451f704be8656, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730988898992 2024-11-07T14:15:03,735 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa6e27e6273a4e54be10fbb0fa3d741c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730988898303 2024-11-07T14:15:03,735 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 63d5c244c4944b7d8021611a71d417b3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730988899332 2024-11-07T14:15:03,735 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 332af80fa04e4633804e88a58fe28b6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730988898992 2024-11-07T14:15:03,735 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d32a682af14c95976fbc9857d10c35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730988900504 2024-11-07T14:15:03,735 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1514b23cca74281b46e3d712e3a40d2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730988899332 2024-11-07T14:15:03,735 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting fb688aaee4ed4d9a867513420694862d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730988901654 2024-11-07T14:15:03,735 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fda8b072ed54087876043c8b3ce905d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730988900504 2024-11-07T14:15:03,736 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 943e615800f247fc9ef7231011ff2d54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730988902819 2024-11-07T14:15:03,736 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713f410b216c46f1a66f23bad7813e32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730988901654 2024-11-07T14:15:03,736 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f069f50beee42b4b7479452d89e0b3e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730988902819 2024-11-07T14:15:03,748 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:03,749 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/31b5778a430a4ad2a5857389cf27d8e3 is 50, key is test_row_0/B:col10/1730988902819/Put/seqid=0 2024-11-07T14:15:03,749 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#234 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:03,750 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/6e3dc1acff86419d956cb4869a250dfc is 50, key is test_row_0/A:col10/1730988902819/Put/seqid=0 2024-11-07T14:15:03,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742098_1274 (size=12663) 2024-11-07T14:15:03,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742099_1275 (size=12663) 2024-11-07T14:15:03,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T14:15:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,797 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:15:03,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/ca82634f16cf4abb8c0847acb1e8c1c0 is 50, key is test_row_0/A:col10/1730988902879/Put/seqid=0 2024-11-07T14:15:03,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742100_1276 (size=12151) 2024-11-07T14:15:03,806 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/ca82634f16cf4abb8c0847acb1e8c1c0 2024-11-07T14:15:03,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/57ff16f3362247bcae282d11186062a1 is 50, key is test_row_0/B:col10/1730988902879/Put/seqid=0 2024-11-07T14:15:03,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742101_1277 (size=12151) 2024-11-07T14:15:03,821 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/57ff16f3362247bcae282d11186062a1 2024-11-07T14:15:03,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/e2d561d17c634d6fa3211aa8bf0f541d is 50, key is test_row_0/C:col10/1730988902879/Put/seqid=0 2024-11-07T14:15:03,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742102_1278 (size=12151) 2024-11-07T14:15:03,834 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/e2d561d17c634d6fa3211aa8bf0f541d 2024-11-07T14:15:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/ca82634f16cf4abb8c0847acb1e8c1c0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/ca82634f16cf4abb8c0847acb1e8c1c0 2024-11-07T14:15:03,842 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/ca82634f16cf4abb8c0847acb1e8c1c0, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T14:15:03,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/57ff16f3362247bcae282d11186062a1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/57ff16f3362247bcae282d11186062a1 2024-11-07T14:15:03,849 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/57ff16f3362247bcae282d11186062a1, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T14:15:03,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/e2d561d17c634d6fa3211aa8bf0f541d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e2d561d17c634d6fa3211aa8bf0f541d 2024-11-07T14:15:03,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e2d561d17c634d6fa3211aa8bf0f541d, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T14:15:03,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 1d800a645cbbc59d7375bdd5295bbe51 in 57ms, sequenceid=249, compaction requested=true 2024-11-07T14:15:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-07T14:15:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-07T14:15:03,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T14:15:03,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 984 msec 2024-11-07T14:15:03,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 995 msec 2024-11-07T14:15:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T14:15:04,003 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:04,003 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-07T14:15:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T14:15:04,006 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:04,007 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:04,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:04,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:15:04,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/c4168668c03243c0823af4fc9e36c55d is 50, key is test_row_0/A:col10/1730988904026/Put/seqid=0 2024-11-07T14:15:04,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742103_1279 (size=14691) 2024-11-07T14:15:04,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988964065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988964066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988964067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988964068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988964070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T14:15:04,158 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/31b5778a430a4ad2a5857389cf27d8e3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/31b5778a430a4ad2a5857389cf27d8e3 2024-11-07T14:15:04,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,164 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into 31b5778a430a4ad2a5857389cf27d8e3(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:04,164 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:04,164 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=10, startTime=1730988903732; duration=0sec 2024-11-07T14:15:04,165 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:04,165 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:15:04,165 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-07T14:15:04,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 82921 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-07T14:15:04,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:15:04,169 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,169 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/26b250f8523a4c1f87b26ac0530ed63a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f0a46c1183324eb68780a81be00ca770, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f9ca0aaf3994b3ba4f10462ead14109, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f321ee3facbd4d2faf442332f92a758f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8bd56f024f1446198fce6b8fb39efa2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d66b1b6afff1438eb2ce375ff81b9289, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e2d561d17c634d6fa3211aa8bf0f541d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=81.0 K 2024-11-07T14:15:04,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b250f8523a4c1f87b26ac0530ed63a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730988898303 2024-11-07T14:15:04,170 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a46c1183324eb68780a81be00ca770, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730988898992 2024-11-07T14:15:04,170 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f9ca0aaf3994b3ba4f10462ead14109, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730988899332 2024-11-07T14:15:04,171 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f321ee3facbd4d2faf442332f92a758f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730988900504 2024-11-07T14:15:04,171 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c8bd56f024f1446198fce6b8fb39efa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1730988901654 2024-11-07T14:15:04,172 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/6e3dc1acff86419d956cb4869a250dfc as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/6e3dc1acff86419d956cb4869a250dfc 2024-11-07T14:15:04,172 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d66b1b6afff1438eb2ce375ff81b9289, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730988902819 2024-11-07T14:15:04,172 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d561d17c634d6fa3211aa8bf0f541d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730988902875 2024-11-07T14:15:04,178 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into 6e3dc1acff86419d956cb4869a250dfc(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:04,178 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:04,178 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=10, startTime=1730988903732; duration=0sec 2024-11-07T14:15:04,178 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:04,178 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:15:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988964182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988964182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988964182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988964182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988964189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,193 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:04,193 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/d52c58079c7d4ee99a403d112977a285 is 50, key is test_row_0/C:col10/1730988902879/Put/seqid=0 2024-11-07T14:15:04,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742104_1280 (size=12697) 2024-11-07T14:15:04,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T14:15:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988964392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988964392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988964392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988964392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988964392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/c4168668c03243c0823af4fc9e36c55d 2024-11-07T14:15:04,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7019a6e3be53431c96113760bfdb5014 is 50, key is test_row_0/B:col10/1730988904026/Put/seqid=0 2024-11-07T14:15:04,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742105_1281 (size=12251) 2024-11-07T14:15:04,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:04,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,602 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/d52c58079c7d4ee99a403d112977a285 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d52c58079c7d4ee99a403d112977a285 2024-11-07T14:15:04,607 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into d52c58079c7d4ee99a403d112977a285(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:04,607 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:04,607 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=9, startTime=1730988903732; duration=0sec 2024-11-07T14:15:04,607 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:04,607 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:15:04,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T14:15:04,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988964702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988964702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988964702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988964702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988964702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:04,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:04,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:04,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7019a6e3be53431c96113760bfdb5014 2024-11-07T14:15:04,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/e841f56199b5498fa339ba2504b6d446 is 50, key is test_row_0/C:col10/1730988904026/Put/seqid=0 2024-11-07T14:15:04,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742106_1282 (size=12251) 2024-11-07T14:15:04,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:04,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:04,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:05,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:05,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T14:15:05,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988965213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:05,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988965213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:05,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988965213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:05,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988965213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:05,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988965213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:05,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:05,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:05,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/e841f56199b5498fa339ba2504b6d446 2024-11-07T14:15:05,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/c4168668c03243c0823af4fc9e36c55d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/c4168668c03243c0823af4fc9e36c55d 2024-11-07T14:15:05,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/c4168668c03243c0823af4fc9e36c55d, entries=200, sequenceid=261, filesize=14.3 K 2024-11-07T14:15:05,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7019a6e3be53431c96113760bfdb5014 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7019a6e3be53431c96113760bfdb5014 2024-11-07T14:15:05,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7019a6e3be53431c96113760bfdb5014, entries=150, sequenceid=261, filesize=12.0 K 2024-11-07T14:15:05,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/e841f56199b5498fa339ba2504b6d446 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e841f56199b5498fa339ba2504b6d446 2024-11-07T14:15:05,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e841f56199b5498fa339ba2504b6d446, entries=150, sequenceid=261, filesize=12.0 K 2024-11-07T14:15:05,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1d800a645cbbc59d7375bdd5295bbe51 in 1270ms, sequenceid=261, compaction requested=true 2024-11-07T14:15:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:05,305 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:05,305 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:05,306 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:05,306 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:05,306 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:15:05,306 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:15:05,306 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:05,306 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:05,306 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/6e3dc1acff86419d956cb4869a250dfc, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/ca82634f16cf4abb8c0847acb1e8c1c0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/c4168668c03243c0823af4fc9e36c55d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=38.6 K 2024-11-07T14:15:05,306 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/31b5778a430a4ad2a5857389cf27d8e3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/57ff16f3362247bcae282d11186062a1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7019a6e3be53431c96113760bfdb5014] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=36.2 K 2024-11-07T14:15:05,307 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e3dc1acff86419d956cb4869a250dfc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730988902819 2024-11-07T14:15:05,307 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 31b5778a430a4ad2a5857389cf27d8e3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730988902819 2024-11-07T14:15:05,307 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ff16f3362247bcae282d11186062a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730988902875 2024-11-07T14:15:05,307 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca82634f16cf4abb8c0847acb1e8c1c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730988902875 2024-11-07T14:15:05,307 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 7019a6e3be53431c96113760bfdb5014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1730988904026 2024-11-07T14:15:05,307 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4168668c03243c0823af4fc9e36c55d, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1730988904026 2024-11-07T14:15:05,329 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:05,330 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/032708732e374a899f56630790056c6e is 50, key is test_row_0/B:col10/1730988904026/Put/seqid=0 2024-11-07T14:15:05,333 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:05,333 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/12d6e4994e704211859ca02488b37055 is 50, key is test_row_0/A:col10/1730988904026/Put/seqid=0 2024-11-07T14:15:05,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742107_1283 (size=12865) 2024-11-07T14:15:05,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742108_1284 (size=12865) 2024-11-07T14:15:05,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T14:15:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:05,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:15:05,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/59e3365cf8b24fe8aa30035223a164e1 is 50, key is test_row_0/A:col10/1730988904068/Put/seqid=0 2024-11-07T14:15:05,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742109_1285 (size=12301) 2024-11-07T14:15:05,398 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/59e3365cf8b24fe8aa30035223a164e1 2024-11-07T14:15:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/18f5dfacf35c4dde8995330b9b5cf96a is 50, key is test_row_0/B:col10/1730988904068/Put/seqid=0 2024-11-07T14:15:05,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742110_1286 (size=12301) 2024-11-07T14:15:05,762 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/032708732e374a899f56630790056c6e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/032708732e374a899f56630790056c6e 2024-11-07T14:15:05,771 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into 032708732e374a899f56630790056c6e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:05,771 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:05,771 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988905305; duration=0sec 2024-11-07T14:15:05,771 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:05,771 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:15:05,773 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T14:15:05,773 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T14:15:05,773 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T14:15:05,774 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. because compaction request was cancelled 2024-11-07T14:15:05,774 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:15:05,794 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/12d6e4994e704211859ca02488b37055 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/12d6e4994e704211859ca02488b37055 2024-11-07T14:15:05,800 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into 12d6e4994e704211859ca02488b37055(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:05,801 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:05,801 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988905305; duration=0sec 2024-11-07T14:15:05,801 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:05,801 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:15:05,810 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/18f5dfacf35c4dde8995330b9b5cf96a 2024-11-07T14:15:05,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/0dd6f89f19cd4425be0a347f32473ebd is 50, key is test_row_0/C:col10/1730988904068/Put/seqid=0 2024-11-07T14:15:05,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742111_1287 (size=12301) 2024-11-07T14:15:05,855 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/0dd6f89f19cd4425be0a347f32473ebd 2024-11-07T14:15:05,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/59e3365cf8b24fe8aa30035223a164e1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/59e3365cf8b24fe8aa30035223a164e1 2024-11-07T14:15:05,868 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/59e3365cf8b24fe8aa30035223a164e1, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T14:15:05,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/18f5dfacf35c4dde8995330b9b5cf96a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/18f5dfacf35c4dde8995330b9b5cf96a 2024-11-07T14:15:05,883 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/18f5dfacf35c4dde8995330b9b5cf96a, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T14:15:05,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/0dd6f89f19cd4425be0a347f32473ebd as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/0dd6f89f19cd4425be0a347f32473ebd 2024-11-07T14:15:05,911 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/0dd6f89f19cd4425be0a347f32473ebd, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T14:15:05,914 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 1d800a645cbbc59d7375bdd5295bbe51 in 525ms, sequenceid=288, compaction requested=true 2024-11-07T14:15:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-07T14:15:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-07T14:15:05,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T14:15:05,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-11-07T14:15:05,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.9390 sec 2024-11-07T14:15:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T14:15:06,153 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:06,153 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-07T14:15:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T14:15:06,160 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:06,161 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:06,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:06,253 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:06,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:15:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T14:15:06,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d0b000c2124e47d9800731bc2f38cb59 is 50, key is test_row_0/A:col10/1730988906246/Put/seqid=0 2024-11-07T14:15:06,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742112_1288 (size=9857) 2024-11-07T14:15:06,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d0b000c2124e47d9800731bc2f38cb59 2024-11-07T14:15:06,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/25b7bcbcb6ad452b878bfb55e21e9bab is 50, key is test_row_0/B:col10/1730988906246/Put/seqid=0 2024-11-07T14:15:06,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742113_1289 (size=9857) 2024-11-07T14:15:06,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988966302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988966303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T14:15:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988966308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988966309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988966309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988966413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988966422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988966422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988966423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988966423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T14:15:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T14:15:06,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T14:15:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988966632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988966632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988966633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988966633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988966633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/25b7bcbcb6ad452b878bfb55e21e9bab 2024-11-07T14:15:06,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4f04105c6cb8444296aa7caefb7a71f3 is 50, key is test_row_0/C:col10/1730988906246/Put/seqid=0 2024-11-07T14:15:06,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742114_1290 (size=9857) 2024-11-07T14:15:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T14:15:06,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T14:15:06,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:06,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T14:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:06,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:06,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988966942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988966942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988966942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988966943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988966943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T14:15:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:07,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4f04105c6cb8444296aa7caefb7a71f3 2024-11-07T14:15:07,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d0b000c2124e47d9800731bc2f38cb59 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0b000c2124e47d9800731bc2f38cb59 2024-11-07T14:15:07,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0b000c2124e47d9800731bc2f38cb59, entries=100, sequenceid=304, filesize=9.6 K 2024-11-07T14:15:07,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/25b7bcbcb6ad452b878bfb55e21e9bab as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/25b7bcbcb6ad452b878bfb55e21e9bab 2024-11-07T14:15:07,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/25b7bcbcb6ad452b878bfb55e21e9bab, entries=100, sequenceid=304, filesize=9.6 K 2024-11-07T14:15:07,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4f04105c6cb8444296aa7caefb7a71f3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f04105c6cb8444296aa7caefb7a71f3 2024-11-07T14:15:07,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f04105c6cb8444296aa7caefb7a71f3, entries=100, sequenceid=304, filesize=9.6 K 2024-11-07T14:15:07,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1d800a645cbbc59d7375bdd5295bbe51 in 941ms, sequenceid=304, compaction requested=true 2024-11-07T14:15:07,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:07,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:07,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:07,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:07,194 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:07,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:07,194 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:07,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:07,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:07,195 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:07,195 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:07,195 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:15:07,195 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:15:07,195 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,195 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,195 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/12d6e4994e704211859ca02488b37055, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/59e3365cf8b24fe8aa30035223a164e1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0b000c2124e47d9800731bc2f38cb59] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=34.2 K 2024-11-07T14:15:07,195 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/032708732e374a899f56630790056c6e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/18f5dfacf35c4dde8995330b9b5cf96a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/25b7bcbcb6ad452b878bfb55e21e9bab] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=34.2 K 2024-11-07T14:15:07,196 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d6e4994e704211859ca02488b37055, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1730988904026 2024-11-07T14:15:07,196 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 032708732e374a899f56630790056c6e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1730988904026 2024-11-07T14:15:07,196 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f5dfacf35c4dde8995330b9b5cf96a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988904065 2024-11-07T14:15:07,196 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e3365cf8b24fe8aa30035223a164e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988904065 2024-11-07T14:15:07,196 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 25b7bcbcb6ad452b878bfb55e21e9bab, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730988906246 2024-11-07T14:15:07,196 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0b000c2124e47d9800731bc2f38cb59, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730988906246 2024-11-07T14:15:07,203 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:07,204 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/214575fec3114c97b552e268e4699030 is 50, key is test_row_0/B:col10/1730988906246/Put/seqid=0 2024-11-07T14:15:07,211 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:07,211 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/e82eeab45be3433584dfdfe61314dcfd is 50, key is test_row_0/A:col10/1730988906246/Put/seqid=0 2024-11-07T14:15:07,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742115_1291 (size=13017) 2024-11-07T14:15:07,226 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/214575fec3114c97b552e268e4699030 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/214575fec3114c97b552e268e4699030 2024-11-07T14:15:07,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742116_1292 (size=13017) 2024-11-07T14:15:07,237 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into 214575fec3114c97b552e268e4699030(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:07,237 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:07,237 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988907194; duration=0sec 2024-11-07T14:15:07,237 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:07,237 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:15:07,237 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:07,238 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/e82eeab45be3433584dfdfe61314dcfd as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e82eeab45be3433584dfdfe61314dcfd 2024-11-07T14:15:07,239 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47106 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:07,239 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:15:07,239 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,239 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d52c58079c7d4ee99a403d112977a285, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e841f56199b5498fa339ba2504b6d446, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/0dd6f89f19cd4425be0a347f32473ebd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f04105c6cb8444296aa7caefb7a71f3] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=46.0 K 2024-11-07T14:15:07,240 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d52c58079c7d4ee99a403d112977a285, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730988902875 2024-11-07T14:15:07,240 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e841f56199b5498fa339ba2504b6d446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1730988904026 2024-11-07T14:15:07,241 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd6f89f19cd4425be0a347f32473ebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988904065 2024-11-07T14:15:07,243 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f04105c6cb8444296aa7caefb7a71f3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730988906246 2024-11-07T14:15:07,246 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into e82eeab45be3433584dfdfe61314dcfd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:07,246 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:07,246 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988907194; duration=0sec 2024-11-07T14:15:07,246 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:07,246 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:15:07,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T14:15:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T14:15:07,272 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:07,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/a8d768b77644438eb6a8912d7d8b5670 is 50, key is test_row_0/A:col10/1730988906307/Put/seqid=0 2024-11-07T14:15:07,273 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/aa5bc85072ab4d8c88b29e4aff85015d is 50, key is test_row_0/C:col10/1730988906246/Put/seqid=0 2024-11-07T14:15:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T14:15:07,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742117_1293 (size=12301) 2024-11-07T14:15:07,321 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/a8d768b77644438eb6a8912d7d8b5670 2024-11-07T14:15:07,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742118_1294 (size=12983) 2024-11-07T14:15:07,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7877cf9d71194ad696911a22bf42990f is 50, key is test_row_0/B:col10/1730988906307/Put/seqid=0 2024-11-07T14:15:07,336 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/aa5bc85072ab4d8c88b29e4aff85015d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/aa5bc85072ab4d8c88b29e4aff85015d 2024-11-07T14:15:07,342 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into aa5bc85072ab4d8c88b29e4aff85015d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:07,342 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:07,342 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=12, startTime=1730988907194; duration=0sec 2024-11-07T14:15:07,342 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:07,342 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:15:07,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742119_1295 (size=12301) 2024-11-07T14:15:07,359 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7877cf9d71194ad696911a22bf42990f 2024-11-07T14:15:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/526ff8cc4ba74e8b8bd3f21facc4c2ae is 50, key is test_row_0/C:col10/1730988906307/Put/seqid=0 2024-11-07T14:15:07,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742120_1296 (size=12301) 2024-11-07T14:15:07,377 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/526ff8cc4ba74e8b8bd3f21facc4c2ae 2024-11-07T14:15:07,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/a8d768b77644438eb6a8912d7d8b5670 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/a8d768b77644438eb6a8912d7d8b5670 2024-11-07T14:15:07,387 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/a8d768b77644438eb6a8912d7d8b5670, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T14:15:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7877cf9d71194ad696911a22bf42990f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7877cf9d71194ad696911a22bf42990f 2024-11-07T14:15:07,392 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7877cf9d71194ad696911a22bf42990f, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T14:15:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/526ff8cc4ba74e8b8bd3f21facc4c2ae as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/526ff8cc4ba74e8b8bd3f21facc4c2ae 2024-11-07T14:15:07,398 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/526ff8cc4ba74e8b8bd3f21facc4c2ae, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T14:15:07,399 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 1d800a645cbbc59d7375bdd5295bbe51 in 149ms, sequenceid=328, compaction requested=false 2024-11-07T14:15:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-07T14:15:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-07T14:15:07,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-07T14:15:07,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2400 sec 2024-11-07T14:15:07,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.2500 sec 2024-11-07T14:15:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:07,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:07,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:15:07,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/83efd4ff7f65480595b566fc3741b5de is 50, key is test_row_0/A:col10/1730988907457/Put/seqid=0 2024-11-07T14:15:07,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742121_1297 (size=9857) 2024-11-07T14:15:07,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/83efd4ff7f65480595b566fc3741b5de 2024-11-07T14:15:07,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988967491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988967491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988967495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988967497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988967499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/0ea81ef433f245f5a8312d27fb9d9141 is 50, key is test_row_0/B:col10/1730988907457/Put/seqid=0 2024-11-07T14:15:07,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742122_1298 (size=9857) 2024-11-07T14:15:07,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/0ea81ef433f245f5a8312d27fb9d9141 2024-11-07T14:15:07,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c5d71568f93446bd8cd76c04c13805e1 is 50, key is test_row_0/C:col10/1730988907457/Put/seqid=0 2024-11-07T14:15:07,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742123_1299 (size=9857) 2024-11-07T14:15:07,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c5d71568f93446bd8cd76c04c13805e1 2024-11-07T14:15:07,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/83efd4ff7f65480595b566fc3741b5de as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/83efd4ff7f65480595b566fc3741b5de 2024-11-07T14:15:07,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/83efd4ff7f65480595b566fc3741b5de, entries=100, sequenceid=340, filesize=9.6 K 2024-11-07T14:15:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/0ea81ef433f245f5a8312d27fb9d9141 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/0ea81ef433f245f5a8312d27fb9d9141 2024-11-07T14:15:07,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/0ea81ef433f245f5a8312d27fb9d9141, entries=100, sequenceid=340, filesize=9.6 K 2024-11-07T14:15:07,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/c5d71568f93446bd8cd76c04c13805e1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c5d71568f93446bd8cd76c04c13805e1 2024-11-07T14:15:07,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c5d71568f93446bd8cd76c04c13805e1, entries=100, sequenceid=340, filesize=9.6 K 2024-11-07T14:15:07,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1d800a645cbbc59d7375bdd5295bbe51 in 117ms, sequenceid=340, compaction requested=true 2024-11-07T14:15:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:07,576 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:07,576 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:07,578 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:07,578 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:07,578 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:15:07,578 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:15:07,578 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,578 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,578 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/214575fec3114c97b552e268e4699030, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7877cf9d71194ad696911a22bf42990f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/0ea81ef433f245f5a8312d27fb9d9141] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=34.4 K 2024-11-07T14:15:07,578 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e82eeab45be3433584dfdfe61314dcfd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/a8d768b77644438eb6a8912d7d8b5670, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/83efd4ff7f65480595b566fc3741b5de] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=34.4 K 2024-11-07T14:15:07,579 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82eeab45be3433584dfdfe61314dcfd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730988904065 2024-11-07T14:15:07,579 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 214575fec3114c97b552e268e4699030, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730988904065 2024-11-07T14:15:07,579 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 7877cf9d71194ad696911a22bf42990f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988906294 2024-11-07T14:15:07,580 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d768b77644438eb6a8912d7d8b5670, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988906294 2024-11-07T14:15:07,580 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83efd4ff7f65480595b566fc3741b5de, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730988907457 2024-11-07T14:15:07,580 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ea81ef433f245f5a8312d27fb9d9141, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730988907457 2024-11-07T14:15:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:07,604 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:07,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T14:15:07,604 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:07,605 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d9e08a1cc8aa45eb95d663982b204881 is 50, key is test_row_0/A:col10/1730988907457/Put/seqid=0 2024-11-07T14:15:07,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988967614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988967614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,621 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:07,621 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/4496ecdd743845ed8e289aca599c9958 is 50, key is test_row_0/B:col10/1730988907457/Put/seqid=0 2024-11-07T14:15:07,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/dda6c659153f496a94f7b78042bfba1e is 50, key is test_row_0/A:col10/1730988907497/Put/seqid=0 2024-11-07T14:15:07,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988967616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988967616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988967616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742124_1300 (size=13119) 2024-11-07T14:15:07,655 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d9e08a1cc8aa45eb95d663982b204881 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d9e08a1cc8aa45eb95d663982b204881 2024-11-07T14:15:07,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742125_1301 (size=14741) 2024-11-07T14:15:07,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742126_1302 (size=13119) 2024-11-07T14:15:07,669 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into d9e08a1cc8aa45eb95d663982b204881(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:07,669 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:07,669 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988907576; duration=0sec 2024-11-07T14:15:07,669 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:07,669 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:15:07,669 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:07,674 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/4496ecdd743845ed8e289aca599c9958 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/4496ecdd743845ed8e289aca599c9958 2024-11-07T14:15:07,674 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:07,674 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:15:07,674 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:07,674 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/aa5bc85072ab4d8c88b29e4aff85015d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/526ff8cc4ba74e8b8bd3f21facc4c2ae, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c5d71568f93446bd8cd76c04c13805e1] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=34.3 K 2024-11-07T14:15:07,675 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa5bc85072ab4d8c88b29e4aff85015d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730988904065 2024-11-07T14:15:07,678 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 526ff8cc4ba74e8b8bd3f21facc4c2ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988906294 2024-11-07T14:15:07,679 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5d71568f93446bd8cd76c04c13805e1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730988907457 2024-11-07T14:15:07,687 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into 4496ecdd743845ed8e289aca599c9958(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:07,687 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:07,687 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988907576; duration=0sec 2024-11-07T14:15:07,687 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:07,687 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:15:07,690 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:07,690 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/70f2116436e448809c3e0a5d060d5224 is 50, key is test_row_0/C:col10/1730988907457/Put/seqid=0 2024-11-07T14:15:07,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742127_1303 (size=13085) 2024-11-07T14:15:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988967722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988967722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988967732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988967732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988967732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988967932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988967932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988967942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988967942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:07,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:07,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988967942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:07,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:07,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/dda6c659153f496a94f7b78042bfba1e 2024-11-07T14:15:08,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7c0c8f766133408f8cf575920a1ef89d is 50, key is test_row_0/B:col10/1730988907497/Put/seqid=0 2024-11-07T14:15:08,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742128_1304 (size=12301) 2024-11-07T14:15:08,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7c0c8f766133408f8cf575920a1ef89d 2024-11-07T14:15:08,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/de8e2b9eaa094b498c81c448948d04b0 is 50, key is test_row_0/C:col10/1730988907497/Put/seqid=0 2024-11-07T14:15:08,104 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/70f2116436e448809c3e0a5d060d5224 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/70f2116436e448809c3e0a5d060d5224 2024-11-07T14:15:08,112 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into 70f2116436e448809c3e0a5d060d5224(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:08,112 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:08,112 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=13, startTime=1730988907577; duration=0sec 2024-11-07T14:15:08,112 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:08,112 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:15:08,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742129_1305 (size=12301) 2024-11-07T14:15:08,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/de8e2b9eaa094b498c81c448948d04b0 2024-11-07T14:15:08,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/dda6c659153f496a94f7b78042bfba1e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/dda6c659153f496a94f7b78042bfba1e 2024-11-07T14:15:08,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/dda6c659153f496a94f7b78042bfba1e, entries=200, sequenceid=367, filesize=14.4 K 2024-11-07T14:15:08,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7c0c8f766133408f8cf575920a1ef89d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7c0c8f766133408f8cf575920a1ef89d 2024-11-07T14:15:08,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7c0c8f766133408f8cf575920a1ef89d, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T14:15:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/de8e2b9eaa094b498c81c448948d04b0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/de8e2b9eaa094b498c81c448948d04b0 2024-11-07T14:15:08,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/de8e2b9eaa094b498c81c448948d04b0, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T14:15:08,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 1d800a645cbbc59d7375bdd5295bbe51 in 553ms, sequenceid=367, compaction requested=false 2024-11-07T14:15:08,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:08,246 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:08,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:15:08,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/72c9c75e285349cf9dbaef814fc324c9 is 50, key is test_row_0/A:col10/1730988907614/Put/seqid=0 2024-11-07T14:15:08,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742130_1306 (size=12301) 2024-11-07T14:15:08,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988968287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988968288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988968290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988968291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988968291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T14:15:08,303 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:08,303 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-07T14:15:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T14:15:08,305 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:08,306 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:08,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:08,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988968402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988968402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988968402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988968402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988968402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T14:15:08,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T14:15:08,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:08,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:08,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:08,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:08,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:08,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988968611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988968612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988968612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988968612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988968612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T14:15:08,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:08,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:08,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:08,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:08,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T14:15:08,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/72c9c75e285349cf9dbaef814fc324c9 2024-11-07T14:15:08,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/e2a3348634604c558fe4ae5271427fd2 is 50, key is test_row_0/B:col10/1730988907614/Put/seqid=0 2024-11-07T14:15:08,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742131_1307 (size=12301) 2024-11-07T14:15:08,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/e2a3348634604c558fe4ae5271427fd2 2024-11-07T14:15:08,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/30ba41c3240748bcbcfbb6490f59048a is 50, key is test_row_0/C:col10/1730988907614/Put/seqid=0 2024-11-07T14:15:08,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742132_1308 (size=12301) 2024-11-07T14:15:08,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/30ba41c3240748bcbcfbb6490f59048a 2024-11-07T14:15:08,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/72c9c75e285349cf9dbaef814fc324c9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/72c9c75e285349cf9dbaef814fc324c9 2024-11-07T14:15:08,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/72c9c75e285349cf9dbaef814fc324c9, entries=150, sequenceid=382, filesize=12.0 K 2024-11-07T14:15:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/e2a3348634604c558fe4ae5271427fd2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e2a3348634604c558fe4ae5271427fd2 2024-11-07T14:15:08,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e2a3348634604c558fe4ae5271427fd2, entries=150, sequenceid=382, filesize=12.0 K 2024-11-07T14:15:08,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/30ba41c3240748bcbcfbb6490f59048a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/30ba41c3240748bcbcfbb6490f59048a 2024-11-07T14:15:08,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/30ba41c3240748bcbcfbb6490f59048a, entries=150, sequenceid=382, filesize=12.0 K 2024-11-07T14:15:08,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1d800a645cbbc59d7375bdd5295bbe51 in 453ms, sequenceid=382, compaction requested=true 2024-11-07T14:15:08,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:08,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:08,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:08,699 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:08,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:08,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:08,699 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:08,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:08,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:08,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:08,701 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:08,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:15:08,701 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:15:08,701 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:08,701 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:08,701 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/4496ecdd743845ed8e289aca599c9958, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7c0c8f766133408f8cf575920a1ef89d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e2a3348634604c558fe4ae5271427fd2] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=36.8 K 2024-11-07T14:15:08,701 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d9e08a1cc8aa45eb95d663982b204881, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/dda6c659153f496a94f7b78042bfba1e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/72c9c75e285349cf9dbaef814fc324c9] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=39.2 K 2024-11-07T14:15:08,701 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e08a1cc8aa45eb95d663982b204881, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730988906294 2024-11-07T14:15:08,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4496ecdd743845ed8e289aca599c9958, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730988906294 2024-11-07T14:15:08,702 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting dda6c659153f496a94f7b78042bfba1e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730988907485 2024-11-07T14:15:08,702 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c0c8f766133408f8cf575920a1ef89d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730988907497 2024-11-07T14:15:08,702 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72c9c75e285349cf9dbaef814fc324c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988907614 2024-11-07T14:15:08,702 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e2a3348634604c558fe4ae5271427fd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988907614 2024-11-07T14:15:08,710 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:08,710 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/df90592e805a4278b6395e2df1d30a58 is 50, key is test_row_0/B:col10/1730988907614/Put/seqid=0 2024-11-07T14:15:08,714 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:08,714 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/84be3457045e428ea2bfd6e436d2146d is 50, key is test_row_0/A:col10/1730988907614/Put/seqid=0 2024-11-07T14:15:08,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742133_1309 (size=13221) 2024-11-07T14:15:08,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742134_1310 (size=13221) 2024-11-07T14:15:08,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T14:15:08,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:08,772 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:15:08,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/9e4c633db7a0402db7726b8ee12b3424 is 50, key is test_row_0/A:col10/1730988908286/Put/seqid=0 2024-11-07T14:15:08,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742135_1311 (size=12301) 2024-11-07T14:15:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:08,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T14:15:08,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988968928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988968930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988968931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988968932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988968933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988969042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988969042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988969042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988969042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988969042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,124 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/df90592e805a4278b6395e2df1d30a58 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/df90592e805a4278b6395e2df1d30a58 2024-11-07T14:15:09,129 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into df90592e805a4278b6395e2df1d30a58(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:09,129 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:09,129 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988908699; duration=0sec 2024-11-07T14:15:09,129 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:09,129 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:15:09,129 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:09,130 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:09,130 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:15:09,130 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:09,130 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/70f2116436e448809c3e0a5d060d5224, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/de8e2b9eaa094b498c81c448948d04b0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/30ba41c3240748bcbcfbb6490f59048a] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=36.8 K 2024-11-07T14:15:09,131 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 70f2116436e448809c3e0a5d060d5224, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1730988906294 2024-11-07T14:15:09,131 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting de8e2b9eaa094b498c81c448948d04b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730988907497 2024-11-07T14:15:09,131 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ba41c3240748bcbcfbb6490f59048a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988907614 2024-11-07T14:15:09,136 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/84be3457045e428ea2bfd6e436d2146d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/84be3457045e428ea2bfd6e436d2146d 2024-11-07T14:15:09,140 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:09,141 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/044031b9e07c4e71aa83fb7c51d97a21 is 50, key is test_row_0/C:col10/1730988907614/Put/seqid=0 2024-11-07T14:15:09,142 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into 84be3457045e428ea2bfd6e436d2146d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:09,142 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:09,142 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988908699; duration=0sec 2024-11-07T14:15:09,142 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:09,142 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:15:09,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742136_1312 (size=13187) 2024-11-07T14:15:09,153 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/044031b9e07c4e71aa83fb7c51d97a21 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/044031b9e07c4e71aa83fb7c51d97a21 2024-11-07T14:15:09,158 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into 044031b9e07c4e71aa83fb7c51d97a21(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:09,158 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:09,158 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=13, startTime=1730988908699; duration=0sec 2024-11-07T14:15:09,158 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:09,159 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:15:09,193 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/9e4c633db7a0402db7726b8ee12b3424 2024-11-07T14:15:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/16866c29e9ac4141973c64dd65c16890 is 50, key is test_row_0/B:col10/1730988908286/Put/seqid=0 2024-11-07T14:15:09,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742137_1313 (size=12301) 2024-11-07T14:15:09,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988969252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988969252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988969252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988969252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988969252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T14:15:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988969562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988969562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988969563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988969562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988969563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:09,611 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/16866c29e9ac4141973c64dd65c16890 2024-11-07T14:15:09,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f9fdda16be6c470dbc69f58ad66dbdb2 is 50, key is test_row_0/C:col10/1730988908286/Put/seqid=0 2024-11-07T14:15:09,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742138_1314 (size=12301) 2024-11-07T14:15:09,635 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f9fdda16be6c470dbc69f58ad66dbdb2 2024-11-07T14:15:09,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/9e4c633db7a0402db7726b8ee12b3424 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/9e4c633db7a0402db7726b8ee12b3424 2024-11-07T14:15:09,652 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/9e4c633db7a0402db7726b8ee12b3424, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T14:15:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/16866c29e9ac4141973c64dd65c16890 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/16866c29e9ac4141973c64dd65c16890 2024-11-07T14:15:09,660 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/16866c29e9ac4141973c64dd65c16890, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T14:15:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/f9fdda16be6c470dbc69f58ad66dbdb2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f9fdda16be6c470dbc69f58ad66dbdb2 2024-11-07T14:15:09,669 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f9fdda16be6c470dbc69f58ad66dbdb2, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T14:15:09,670 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1d800a645cbbc59d7375bdd5295bbe51 in 898ms, sequenceid=406, compaction requested=false 2024-11-07T14:15:09,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:09,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:09,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-07T14:15:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-07T14:15:09,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T14:15:09,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3660 sec 2024-11-07T14:15:09,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.3730 sec 2024-11-07T14:15:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:10,074 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:10,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:15:10,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/35f02f955d6249d2b9c8eba99196cdbe is 50, key is test_row_0/A:col10/1730988910075/Put/seqid=0 2024-11-07T14:15:10,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742139_1315 (size=12301) 2024-11-07T14:15:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988970093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988970093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988970097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988970097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988970099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988970202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988970202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988970202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988970202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988970211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988970412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988970412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988970413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988970414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988970423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T14:15:10,454 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:10,455 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-07T14:15:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T14:15:10,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:10,461 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:10,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:10,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/35f02f955d6249d2b9c8eba99196cdbe 2024-11-07T14:15:10,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/63c5aa64d4ef4971aa5e59f85559e215 is 50, key is test_row_0/B:col10/1730988910075/Put/seqid=0 2024-11-07T14:15:10,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742140_1316 (size=12301) 2024-11-07T14:15:10,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/63c5aa64d4ef4971aa5e59f85559e215 2024-11-07T14:15:10,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/7e8182133b484b0a9f37aa5dc8a6bc20 is 50, key is test_row_0/C:col10/1730988910075/Put/seqid=0 2024-11-07T14:15:10,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742141_1317 (size=12301) 2024-11-07T14:15:10,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/7e8182133b484b0a9f37aa5dc8a6bc20 2024-11-07T14:15:10,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/35f02f955d6249d2b9c8eba99196cdbe as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/35f02f955d6249d2b9c8eba99196cdbe 2024-11-07T14:15:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T14:15:10,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/35f02f955d6249d2b9c8eba99196cdbe, entries=150, sequenceid=424, filesize=12.0 K 2024-11-07T14:15:10,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/63c5aa64d4ef4971aa5e59f85559e215 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63c5aa64d4ef4971aa5e59f85559e215 2024-11-07T14:15:10,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63c5aa64d4ef4971aa5e59f85559e215, entries=150, sequenceid=424, filesize=12.0 K 2024-11-07T14:15:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/7e8182133b484b0a9f37aa5dc8a6bc20 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/7e8182133b484b0a9f37aa5dc8a6bc20 2024-11-07T14:15:10,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/7e8182133b484b0a9f37aa5dc8a6bc20, entries=150, sequenceid=424, filesize=12.0 K 2024-11-07T14:15:10,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1d800a645cbbc59d7375bdd5295bbe51 in 509ms, sequenceid=424, compaction requested=true 2024-11-07T14:15:10,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:10,584 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:10,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:10,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:10,584 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:10,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:10,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:10,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:10,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:10,588 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:10,588 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:15:10,588 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:10,588 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:15:10,589 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:10,589 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:10,589 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/84be3457045e428ea2bfd6e436d2146d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/9e4c633db7a0402db7726b8ee12b3424, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/35f02f955d6249d2b9c8eba99196cdbe] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=36.9 K 2024-11-07T14:15:10,589 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/df90592e805a4278b6395e2df1d30a58, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/16866c29e9ac4141973c64dd65c16890, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63c5aa64d4ef4971aa5e59f85559e215] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=36.9 K 2024-11-07T14:15:10,590 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting df90592e805a4278b6395e2df1d30a58, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988907614 2024-11-07T14:15:10,590 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84be3457045e428ea2bfd6e436d2146d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988907614 2024-11-07T14:15:10,592 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 16866c29e9ac4141973c64dd65c16890, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730988908286 2024-11-07T14:15:10,592 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e4c633db7a0402db7726b8ee12b3424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730988908286 2024-11-07T14:15:10,593 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35f02f955d6249d2b9c8eba99196cdbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1730988908926 2024-11-07T14:15:10,593 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 63c5aa64d4ef4971aa5e59f85559e215, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1730988908926 2024-11-07T14:15:10,605 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:10,605 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/935fcba155bb40c1bf9674c1e12e1c92 is 50, key is test_row_0/B:col10/1730988910075/Put/seqid=0 2024-11-07T14:15:10,609 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:10,610 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/0e8e5a8be6d1457188c801d37c0eb8a0 is 50, key is test_row_0/A:col10/1730988910075/Put/seqid=0 2024-11-07T14:15:10,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T14:15:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:10,614 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T14:15:10,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/41679067d583450ebcdb872ac2d22a40 is 50, key is test_row_0/A:col10/1730988910096/Put/seqid=0 2024-11-07T14:15:10,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742142_1318 (size=13323) 2024-11-07T14:15:10,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742143_1319 (size=13323) 2024-11-07T14:15:10,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742144_1320 (size=12301) 2024-11-07T14:15:10,670 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/935fcba155bb40c1bf9674c1e12e1c92 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/935fcba155bb40c1bf9674c1e12e1c92 2024-11-07T14:15:10,678 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into 935fcba155bb40c1bf9674c1e12e1c92(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:10,678 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:10,678 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988910584; duration=0sec 2024-11-07T14:15:10,678 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:10,678 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:15:10,679 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:10,683 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:10,683 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:15:10,683 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:10,683 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/044031b9e07c4e71aa83fb7c51d97a21, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f9fdda16be6c470dbc69f58ad66dbdb2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/7e8182133b484b0a9f37aa5dc8a6bc20] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=36.9 K 2024-11-07T14:15:10,684 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 044031b9e07c4e71aa83fb7c51d97a21, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1730988907614 2024-11-07T14:15:10,684 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f9fdda16be6c470dbc69f58ad66dbdb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730988908286 2024-11-07T14:15:10,685 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e8182133b484b0a9f37aa5dc8a6bc20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1730988908926 2024-11-07T14:15:10,695 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:10,696 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/3b1aeb98f0fd49418e86804937015fff is 50, key is test_row_0/C:col10/1730988910075/Put/seqid=0 2024-11-07T14:15:10,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:10,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742145_1321 (size=13289) 2024-11-07T14:15:10,741 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/3b1aeb98f0fd49418e86804937015fff as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/3b1aeb98f0fd49418e86804937015fff 2024-11-07T14:15:10,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988970740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988970745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988970748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988970749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,754 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into 3b1aeb98f0fd49418e86804937015fff(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:10,754 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:10,754 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=13, startTime=1730988910585; duration=0sec 2024-11-07T14:15:10,754 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:10,754 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:15:10,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988970751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T14:15:10,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988970852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988970853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988970863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988970863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:10,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988970863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988971062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988971062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,067 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/41679067d583450ebcdb872ac2d22a40 2024-11-07T14:15:11,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988971073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988971074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988971074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T14:15:11,090 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/0e8e5a8be6d1457188c801d37c0eb8a0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/0e8e5a8be6d1457188c801d37c0eb8a0 2024-11-07T14:15:11,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/729dc97e362e44bca83e18aa259dd3e1 is 50, key is test_row_0/B:col10/1730988910096/Put/seqid=0 2024-11-07T14:15:11,104 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into 0e8e5a8be6d1457188c801d37c0eb8a0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:11,104 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:11,104 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988910584; duration=0sec 2024-11-07T14:15:11,104 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:11,104 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:15:11,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742146_1322 (size=12301) 2024-11-07T14:15:11,123 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/729dc97e362e44bca83e18aa259dd3e1 2024-11-07T14:15:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/77267a6b29ed44859c2302d0f2b2c9be is 50, key is test_row_0/C:col10/1730988910096/Put/seqid=0 2024-11-07T14:15:11,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742147_1323 (size=12301) 2024-11-07T14:15:11,136 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/77267a6b29ed44859c2302d0f2b2c9be 2024-11-07T14:15:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/41679067d583450ebcdb872ac2d22a40 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/41679067d583450ebcdb872ac2d22a40 2024-11-07T14:15:11,149 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/41679067d583450ebcdb872ac2d22a40, entries=150, sequenceid=445, filesize=12.0 K 2024-11-07T14:15:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/729dc97e362e44bca83e18aa259dd3e1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/729dc97e362e44bca83e18aa259dd3e1 2024-11-07T14:15:11,154 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/729dc97e362e44bca83e18aa259dd3e1, entries=150, sequenceid=445, filesize=12.0 K 2024-11-07T14:15:11,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/77267a6b29ed44859c2302d0f2b2c9be as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/77267a6b29ed44859c2302d0f2b2c9be 2024-11-07T14:15:11,158 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/77267a6b29ed44859c2302d0f2b2c9be, entries=150, sequenceid=445, filesize=12.0 K 2024-11-07T14:15:11,163 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 1d800a645cbbc59d7375bdd5295bbe51 in 549ms, sequenceid=445, compaction requested=false 2024-11-07T14:15:11,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:11,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:11,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-07T14:15:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-07T14:15:11,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T14:15:11,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 704 msec 2024-11-07T14:15:11,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 713 msec 2024-11-07T14:15:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:11,374 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:11,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T14:15:11,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/95d5f97230ae42c784443ec3f08e1341 is 50, key is test_row_0/A:col10/1730988910746/Put/seqid=0 2024-11-07T14:15:11,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988971393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988971393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988971394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988971395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988971399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742148_1324 (size=14741) 2024-11-07T14:15:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988971503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988971503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988971504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988971505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988971512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T14:15:11,592 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:11,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-07T14:15:11,594 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T14:15:11,594 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:11,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T14:15:11,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988971712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988971712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988971712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988971712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988971722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:11,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-07T14:15:11,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:11,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:11,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:11,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:11,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:11,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/95d5f97230ae42c784443ec3f08e1341 2024-11-07T14:15:11,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/9c4033b37adb43a2b3f45759ea5e8826 is 50, key is test_row_0/B:col10/1730988910746/Put/seqid=0 2024-11-07T14:15:11,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742149_1325 (size=12301) 2024-11-07T14:15:11,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/9c4033b37adb43a2b3f45759ea5e8826 2024-11-07T14:15:11,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/ec9c9bf84e4f44889e372096f11e7337 is 50, key is test_row_0/C:col10/1730988910746/Put/seqid=0 2024-11-07T14:15:11,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742150_1326 (size=12301) 2024-11-07T14:15:11,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/ec9c9bf84e4f44889e372096f11e7337 2024-11-07T14:15:11,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/95d5f97230ae42c784443ec3f08e1341 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/95d5f97230ae42c784443ec3f08e1341 2024-11-07T14:15:11,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/95d5f97230ae42c784443ec3f08e1341, entries=200, sequenceid=465, filesize=14.4 K 2024-11-07T14:15:11,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/9c4033b37adb43a2b3f45759ea5e8826 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9c4033b37adb43a2b3f45759ea5e8826 2024-11-07T14:15:11,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9c4033b37adb43a2b3f45759ea5e8826, entries=150, sequenceid=465, filesize=12.0 K 2024-11-07T14:15:11,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/ec9c9bf84e4f44889e372096f11e7337 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/ec9c9bf84e4f44889e372096f11e7337 2024-11-07T14:15:11,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/ec9c9bf84e4f44889e372096f11e7337, entries=150, sequenceid=465, filesize=12.0 K 2024-11-07T14:15:11,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 1d800a645cbbc59d7375bdd5295bbe51 in 521ms, sequenceid=465, compaction requested=true 2024-11-07T14:15:11,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:11,895 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:11,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:11,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:11,896 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:11,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:11,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:11,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:11,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:11,899 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:11,899 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:15:11,900 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:11,900 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:11,900 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:15:11,900 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/0e8e5a8be6d1457188c801d37c0eb8a0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/41679067d583450ebcdb872ac2d22a40, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/95d5f97230ae42c784443ec3f08e1341] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=39.4 K 2024-11-07T14:15:11,900 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:11,900 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/935fcba155bb40c1bf9674c1e12e1c92, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/729dc97e362e44bca83e18aa259dd3e1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9c4033b37adb43a2b3f45759ea5e8826] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=37.0 K 2024-11-07T14:15:11,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-07T14:15:11,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:11,900 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T14:15:11,901 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 935fcba155bb40c1bf9674c1e12e1c92, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1730988908926 2024-11-07T14:15:11,902 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e8e5a8be6d1457188c801d37c0eb8a0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1730988908926 2024-11-07T14:15:11,902 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 729dc97e362e44bca83e18aa259dd3e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1730988910092 2024-11-07T14:15:11,902 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41679067d583450ebcdb872ac2d22a40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1730988910092 2024-11-07T14:15:11,911 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d5f97230ae42c784443ec3f08e1341, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730988910745 2024-11-07T14:15:11,911 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4033b37adb43a2b3f45759ea5e8826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730988910746 2024-11-07T14:15:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T14:15:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d782e4f821974d398582ec555c035831 is 50, key is test_row_0/A:col10/1730988911393/Put/seqid=0 2024-11-07T14:15:11,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742151_1327 (size=12301) 2024-11-07T14:15:11,929 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d782e4f821974d398582ec555c035831 2024-11-07T14:15:11,932 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:11,932 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d0a0fa226b624ce693d3bd88b4bd1c88 is 50, key is test_row_0/A:col10/1730988910746/Put/seqid=0 2024-11-07T14:15:11,932 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:11,932 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7936869fba7944faad9c64fc7f5c4529 is 50, key is test_row_0/B:col10/1730988910746/Put/seqid=0 2024-11-07T14:15:11,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742153_1329 (size=13425) 2024-11-07T14:15:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/8a1be277715a42489b0e1fdc2e26e090 is 50, key is test_row_0/B:col10/1730988911393/Put/seqid=0 2024-11-07T14:15:11,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742152_1328 (size=13425) 2024-11-07T14:15:11,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742154_1330 (size=12301) 2024-11-07T14:15:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:12,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988972039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988972039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988972039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988972044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988972045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988972152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988972152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988972152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988972152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988972153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T14:15:12,357 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d0a0fa226b624ce693d3bd88b4bd1c88 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0a0fa226b624ce693d3bd88b4bd1c88 2024-11-07T14:15:12,363 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into d0a0fa226b624ce693d3bd88b4bd1c88(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:12,363 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:12,363 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/7936869fba7944faad9c64fc7f5c4529 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7936869fba7944faad9c64fc7f5c4529 2024-11-07T14:15:12,363 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988911895; duration=0sec 2024-11-07T14:15:12,364 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:12,364 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:15:12,364 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:12,365 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:12,365 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:15:12,365 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:12,366 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/3b1aeb98f0fd49418e86804937015fff, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/77267a6b29ed44859c2302d0f2b2c9be, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/ec9c9bf84e4f44889e372096f11e7337] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=37.0 K 2024-11-07T14:15:12,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988972362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988972363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988972363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988972363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988972364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,370 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b1aeb98f0fd49418e86804937015fff, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1730988908926 2024-11-07T14:15:12,371 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77267a6b29ed44859c2302d0f2b2c9be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1730988910092 2024-11-07T14:15:12,371 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec9c9bf84e4f44889e372096f11e7337, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730988910746 2024-11-07T14:15:12,377 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/8a1be277715a42489b0e1fdc2e26e090 2024-11-07T14:15:12,378 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into 7936869fba7944faad9c64fc7f5c4529(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:12,378 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:12,378 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988911896; duration=0sec 2024-11-07T14:15:12,378 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:12,378 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:15:12,386 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/9907b2d1ae574345a3ea0e095824cc51 is 50, key is test_row_0/C:col10/1730988911393/Put/seqid=0 2024-11-07T14:15:12,387 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4ce522a593ea47ae9bd3fea94859ca3b is 50, key is test_row_0/C:col10/1730988910746/Put/seqid=0 2024-11-07T14:15:12,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742155_1331 (size=13391) 2024-11-07T14:15:12,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742156_1332 (size=12301) 2024-11-07T14:15:12,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/9907b2d1ae574345a3ea0e095824cc51 2024-11-07T14:15:12,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/d782e4f821974d398582ec555c035831 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d782e4f821974d398582ec555c035831 2024-11-07T14:15:12,420 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d782e4f821974d398582ec555c035831, entries=150, sequenceid=484, filesize=12.0 K 2024-11-07T14:15:12,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/8a1be277715a42489b0e1fdc2e26e090 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8a1be277715a42489b0e1fdc2e26e090 2024-11-07T14:15:12,424 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8a1be277715a42489b0e1fdc2e26e090, entries=150, sequenceid=484, filesize=12.0 K 2024-11-07T14:15:12,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/9907b2d1ae574345a3ea0e095824cc51 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9907b2d1ae574345a3ea0e095824cc51 2024-11-07T14:15:12,434 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9907b2d1ae574345a3ea0e095824cc51, entries=150, sequenceid=484, filesize=12.0 K 2024-11-07T14:15:12,435 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 1d800a645cbbc59d7375bdd5295bbe51 in 535ms, sequenceid=484, compaction requested=false 2024-11-07T14:15:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-07T14:15:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-07T14:15:12,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-07T14:15:12,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 843 msec 2024-11-07T14:15:12,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 847 msec 2024-11-07T14:15:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:12,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:12,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:15:12,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/f6ecc581930a4f5ba196e2a892d468d8 is 50, key is test_row_0/A:col10/1730988912037/Put/seqid=0 2024-11-07T14:15:12,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742157_1333 (size=12301) 2024-11-07T14:15:12,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988972687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988972687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988972688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988972689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988972692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T14:15:12,732 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:12,732 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-07T14:15:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T14:15:12,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:12,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:12,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:12,807 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/4ce522a593ea47ae9bd3fea94859ca3b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4ce522a593ea47ae9bd3fea94859ca3b 2024-11-07T14:15:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988972803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988972803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988972803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988972803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988972804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:12,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:12,818 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into 4ce522a593ea47ae9bd3fea94859ca3b(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:12,818 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:12,818 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=13, startTime=1730988911896; duration=0sec 2024-11-07T14:15:12,818 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:12,818 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:15:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T14:15:12,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T14:15:12,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:12,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:12,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:12,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:12,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988973012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988973012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988973012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988973012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988973013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T14:15:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:13,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T14:15:13,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/f6ecc581930a4f5ba196e2a892d468d8 2024-11-07T14:15:13,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/161ee585bb7c400ab8570512feafe038 is 50, key is test_row_0/B:col10/1730988912037/Put/seqid=0 2024-11-07T14:15:13,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742158_1334 (size=12301) 2024-11-07T14:15:13,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T14:15:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988973323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988973323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988973323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988973323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988973324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T14:15:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T14:15:13,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T14:15:13,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/161ee585bb7c400ab8570512feafe038 2024-11-07T14:15:13,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:13,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/2029ace6d64d4e60b317318058f28a34 is 50, key is test_row_0/C:col10/1730988912037/Put/seqid=0 2024-11-07T14:15:13,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742159_1335 (size=12301) 2024-11-07T14:15:13,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T14:15:13,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:13,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T14:15:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1730988973842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51444 deadline: 1730988973842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51492 deadline: 1730988973842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51486 deadline: 1730988973842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51472 deadline: 1730988973842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T14:15:13,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/2029ace6d64d4e60b317318058f28a34 2024-11-07T14:15:13,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/f6ecc581930a4f5ba196e2a892d468d8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f6ecc581930a4f5ba196e2a892d468d8 2024-11-07T14:15:13,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f6ecc581930a4f5ba196e2a892d468d8, entries=150, sequenceid=504, filesize=12.0 K 2024-11-07T14:15:13,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/161ee585bb7c400ab8570512feafe038 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/161ee585bb7c400ab8570512feafe038 2024-11-07T14:15:13,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/161ee585bb7c400ab8570512feafe038, entries=150, sequenceid=504, filesize=12.0 K 2024-11-07T14:15:13,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/2029ace6d64d4e60b317318058f28a34 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2029ace6d64d4e60b317318058f28a34 2024-11-07T14:15:13,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2029ace6d64d4e60b317318058f28a34, entries=150, sequenceid=504, filesize=12.0 K 2024-11-07T14:15:13,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 1d800a645cbbc59d7375bdd5295bbe51 in 1258ms, sequenceid=504, compaction requested=true 2024-11-07T14:15:13,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:13,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:13,932 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:13,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:13,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:13,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:13,932 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:13,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d800a645cbbc59d7375bdd5295bbe51:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:13,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:13,933 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:13,933 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:13,933 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/A is initiating minor compaction (all files) 2024-11-07T14:15:13,933 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/B is initiating minor compaction (all files) 2024-11-07T14:15:13,933 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/A in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,933 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/B in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,933 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0a0fa226b624ce693d3bd88b4bd1c88, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d782e4f821974d398582ec555c035831, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f6ecc581930a4f5ba196e2a892d468d8] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=37.1 K 2024-11-07T14:15:13,933 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7936869fba7944faad9c64fc7f5c4529, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8a1be277715a42489b0e1fdc2e26e090, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/161ee585bb7c400ab8570512feafe038] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=37.1 K 2024-11-07T14:15:13,934 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 7936869fba7944faad9c64fc7f5c4529, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730988910746 2024-11-07T14:15:13,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a0fa226b624ce693d3bd88b4bd1c88, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730988910746 2024-11-07T14:15:13,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d782e4f821974d398582ec555c035831, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1730988911393 2024-11-07T14:15:13,934 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a1be277715a42489b0e1fdc2e26e090, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1730988911393 2024-11-07T14:15:13,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ecc581930a4f5ba196e2a892d468d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1730988912037 2024-11-07T14:15:13,934 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 161ee585bb7c400ab8570512feafe038, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1730988912037 2024-11-07T14:15:13,941 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#A#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:13,941 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/983bfd3d40644aff9bc50206be1d99f6 is 50, key is test_row_0/A:col10/1730988912037/Put/seqid=0 2024-11-07T14:15:13,941 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#B#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:13,941 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/a5863bc0868a47048c203db748bd3d6d is 50, key is test_row_0/B:col10/1730988912037/Put/seqid=0 2024-11-07T14:15:13,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T14:15:13,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:13,957 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:15:13,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742161_1337 (size=13527) 2024-11-07T14:15:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/4bb9f1e2a25d4184a31ddc1ec45de7ff is 50, key is test_row_0/A:col10/1730988912686/Put/seqid=0 2024-11-07T14:15:13,987 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-07T14:15:13,987 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:13,988 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,988 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:13,989 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-07T14:15:13,989 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-07T14:15:13,989 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:13,989 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:13,989 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,989 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,990 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:13,990 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:13,991 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-07T14:15:13,991 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:13,991 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,991 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:13,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742162_1338 (size=12301) 2024-11-07T14:15:13,992 INFO [Thread-1084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1084. 2024-11-07T14:15:13,992 DEBUG [Thread-1084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:13,993 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,993 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:13,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:13,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742160_1336 (size=13527) 2024-11-07T14:15:14,383 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/a5863bc0868a47048c203db748bd3d6d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/a5863bc0868a47048c203db748bd3d6d 2024-11-07T14:15:14,389 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/B of 1d800a645cbbc59d7375bdd5295bbe51 into a5863bc0868a47048c203db748bd3d6d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:14,389 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:14,389 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/B, priority=13, startTime=1730988913932; duration=0sec 2024-11-07T14:15:14,389 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:14,389 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:B 2024-11-07T14:15:14,389 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:14,390 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:14,390 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 1d800a645cbbc59d7375bdd5295bbe51/C is initiating minor compaction (all files) 2024-11-07T14:15:14,390 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d800a645cbbc59d7375bdd5295bbe51/C in TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:14,390 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4ce522a593ea47ae9bd3fea94859ca3b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9907b2d1ae574345a3ea0e095824cc51, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2029ace6d64d4e60b317318058f28a34] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp, totalSize=37.1 K 2024-11-07T14:15:14,391 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce522a593ea47ae9bd3fea94859ca3b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1730988910746 2024-11-07T14:15:14,391 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9907b2d1ae574345a3ea0e095824cc51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1730988911393 2024-11-07T14:15:14,391 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2029ace6d64d4e60b317318058f28a34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1730988912037 2024-11-07T14:15:14,392 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/4bb9f1e2a25d4184a31ddc1ec45de7ff 2024-11-07T14:15:14,398 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d800a645cbbc59d7375bdd5295bbe51#C#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:14,398 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/012e775888ce4e6d9d4e54acfd6d3f78 is 50, key is test_row_0/C:col10/1730988912037/Put/seqid=0 2024-11-07T14:15:14,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/192ed9e4f5534fed97c678a29cb9d78a is 50, key is test_row_0/B:col10/1730988912686/Put/seqid=0 2024-11-07T14:15:14,401 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/983bfd3d40644aff9bc50206be1d99f6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/983bfd3d40644aff9bc50206be1d99f6 2024-11-07T14:15:14,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742163_1339 (size=13493) 2024-11-07T14:15:14,406 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/A of 1d800a645cbbc59d7375bdd5295bbe51 into 983bfd3d40644aff9bc50206be1d99f6(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:14,406 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:14,406 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/A, priority=13, startTime=1730988913932; duration=0sec 2024-11-07T14:15:14,406 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:14,406 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:A 2024-11-07T14:15:14,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742164_1340 (size=12301) 2024-11-07T14:15:14,808 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/192ed9e4f5534fed97c678a29cb9d78a 2024-11-07T14:15:14,810 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/012e775888ce4e6d9d4e54acfd6d3f78 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/012e775888ce4e6d9d4e54acfd6d3f78 2024-11-07T14:15:14,814 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d800a645cbbc59d7375bdd5295bbe51/C of 1d800a645cbbc59d7375bdd5295bbe51 into 012e775888ce4e6d9d4e54acfd6d3f78(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:14,814 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:14,814 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., storeName=1d800a645cbbc59d7375bdd5295bbe51/C, priority=13, startTime=1730988913932; duration=0sec 2024-11-07T14:15:14,814 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:14,814 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d800a645cbbc59d7375bdd5295bbe51:C 2024-11-07T14:15:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/21c200560c4c41ef8b64ca8de1db55df is 50, key is test_row_0/C:col10/1730988912686/Put/seqid=0 2024-11-07T14:15:14,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742165_1341 (size=12301) 2024-11-07T14:15:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:14,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. as already flushing 2024-11-07T14:15:14,853 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-07T14:15:14,853 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-07T14:15:14,854 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:14,854 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:14,854 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,854 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,854 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-07T14:15:14,854 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,854 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:14,854 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,854 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:14,854 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:14,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:14,855 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-07T14:15:14,855 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:14,855 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,855 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:14,864 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-07T14:15:14,864 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:14,864 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,864 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:14,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T14:15:15,076 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:15:15,219 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/21c200560c4c41ef8b64ca8de1db55df 2024-11-07T14:15:15,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/4bb9f1e2a25d4184a31ddc1ec45de7ff as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4bb9f1e2a25d4184a31ddc1ec45de7ff 2024-11-07T14:15:15,227 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4bb9f1e2a25d4184a31ddc1ec45de7ff, entries=150, sequenceid=523, filesize=12.0 K 2024-11-07T14:15:15,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/192ed9e4f5534fed97c678a29cb9d78a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/192ed9e4f5534fed97c678a29cb9d78a 2024-11-07T14:15:15,233 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/192ed9e4f5534fed97c678a29cb9d78a, entries=150, sequenceid=523, filesize=12.0 K 2024-11-07T14:15:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/21c200560c4c41ef8b64ca8de1db55df as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/21c200560c4c41ef8b64ca8de1db55df 2024-11-07T14:15:15,239 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/21c200560c4c41ef8b64ca8de1db55df, entries=150, sequenceid=523, filesize=12.0 K 2024-11-07T14:15:15,239 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=33.54 KB/34350 for 1d800a645cbbc59d7375bdd5295bbe51 in 1282ms, sequenceid=523, compaction requested=false 2024-11-07T14:15:15,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 1d800a645cbbc59d7375bdd5295bbe51: 2024-11-07T14:15:15,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:15,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-07T14:15:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-07T14:15:15,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T14:15:15,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5070 sec 2024-11-07T14:15:15,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 2.5110 sec 2024-11-07T14:15:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T14:15:16,892 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:16,892 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3472 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3622 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3519 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3486 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3555 2024-11-07T14:15:16,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T14:15:16,893 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T14:15:16,893 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:15:16,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:16,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:16,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:16,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T14:15:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T14:15:16,896 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988916896"}]},"ts":"1730988916896"} 2024-11-07T14:15:16,897 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T14:15:16,897 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T14:15:16,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:15:16,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, UNASSIGN}] 2024-11-07T14:15:16,900 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, UNASSIGN 2024-11-07T14:15:16,900 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=1d800a645cbbc59d7375bdd5295bbe51, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:16,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, UNASSIGN because future has completed 2024-11-07T14:15:16,902 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:15:16,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:15:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T14:15:17,055 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:17,055 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:15:17,055 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 1d800a645cbbc59d7375bdd5295bbe51, disabling compactions & flushes 2024-11-07T14:15:17,055 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:17,055 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:17,055 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. after waiting 0 ms 2024-11-07T14:15:17,055 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:17,055 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing 1d800a645cbbc59d7375bdd5295bbe51 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T14:15:17,059 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/23688201341b4226bf1634990f2a7fce is 50, key is test_row_2/A:col10/1730988914862/Put/seqid=0 2024-11-07T14:15:17,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742166_1342 (size=7415) 2024-11-07T14:15:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T14:15:17,463 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/23688201341b4226bf1634990f2a7fce 2024-11-07T14:15:17,469 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/26711f73590d4dc5ba4b9e21fdddef9e is 50, key is test_row_2/B:col10/1730988914862/Put/seqid=0 2024-11-07T14:15:17,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742167_1343 (size=7415) 2024-11-07T14:15:17,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T14:15:17,872 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/26711f73590d4dc5ba4b9e21fdddef9e 2024-11-07T14:15:17,881 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/45bbb0a5098f43b09e1b4273d69453cf is 50, key is test_row_2/C:col10/1730988914862/Put/seqid=0 2024-11-07T14:15:17,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742168_1344 (size=7415) 2024-11-07T14:15:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T14:15:18,284 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/45bbb0a5098f43b09e1b4273d69453cf 2024-11-07T14:15:18,289 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/A/23688201341b4226bf1634990f2a7fce as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/23688201341b4226bf1634990f2a7fce 2024-11-07T14:15:18,292 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/23688201341b4226bf1634990f2a7fce, entries=50, sequenceid=534, filesize=7.2 K 2024-11-07T14:15:18,293 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/B/26711f73590d4dc5ba4b9e21fdddef9e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/26711f73590d4dc5ba4b9e21fdddef9e 2024-11-07T14:15:18,298 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/26711f73590d4dc5ba4b9e21fdddef9e, entries=50, sequenceid=534, filesize=7.2 K 2024-11-07T14:15:18,300 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/.tmp/C/45bbb0a5098f43b09e1b4273d69453cf as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/45bbb0a5098f43b09e1b4273d69453cf 2024-11-07T14:15:18,304 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/45bbb0a5098f43b09e1b4273d69453cf, entries=50, sequenceid=534, filesize=7.2 K 2024-11-07T14:15:18,305 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1d800a645cbbc59d7375bdd5295bbe51 in 1250ms, sequenceid=534, compaction requested=true 2024-11-07T14:15:18,306 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/732f7ff2475e42519d132f5caf67ae12, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f0eb6484398340dcaf363ca68f260d08, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4ee04698df9d43a6bfb4d4850dfbb6a2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/1a6cbc9f12744dbcbe4c3c2688d3aad7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/61d165453f4e41d1af78233d08ff6f6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/fd4a86f614e44bb185d870632a9919ec, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2ab499e05a774537bb68b34ad2a84aec, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/784f60d52ffc4dc78281072c23e27b37, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/aa6e27e6273a4e54be10fbb0fa3d741c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8662e79f6d914ef4b140fa4f4c6eedba, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/332af80fa04e4633804e88a58fe28b6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e1514b23cca74281b46e3d712e3a40d2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8fda8b072ed54087876043c8b3ce905d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/713f410b216c46f1a66f23bad7813e32, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2f069f50beee42b4b7479452d89e0b3e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/6e3dc1acff86419d956cb4869a250dfc, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/ca82634f16cf4abb8c0847acb1e8c1c0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/c4168668c03243c0823af4fc9e36c55d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/12d6e4994e704211859ca02488b37055, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/59e3365cf8b24fe8aa30035223a164e1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e82eeab45be3433584dfdfe61314dcfd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0b000c2124e47d9800731bc2f38cb59, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/a8d768b77644438eb6a8912d7d8b5670, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d9e08a1cc8aa45eb95d663982b204881, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/83efd4ff7f65480595b566fc3741b5de, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/dda6c659153f496a94f7b78042bfba1e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/84be3457045e428ea2bfd6e436d2146d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/72c9c75e285349cf9dbaef814fc324c9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/9e4c633db7a0402db7726b8ee12b3424, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/0e8e5a8be6d1457188c801d37c0eb8a0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/35f02f955d6249d2b9c8eba99196cdbe, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/41679067d583450ebcdb872ac2d22a40, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/95d5f97230ae42c784443ec3f08e1341, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0a0fa226b624ce693d3bd88b4bd1c88, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d782e4f821974d398582ec555c035831, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f6ecc581930a4f5ba196e2a892d468d8] to archive 2024-11-07T14:15:18,307 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:15:18,309 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/732f7ff2475e42519d132f5caf67ae12 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/732f7ff2475e42519d132f5caf67ae12 2024-11-07T14:15:18,310 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f0eb6484398340dcaf363ca68f260d08 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f0eb6484398340dcaf363ca68f260d08 2024-11-07T14:15:18,311 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4ee04698df9d43a6bfb4d4850dfbb6a2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4ee04698df9d43a6bfb4d4850dfbb6a2 2024-11-07T14:15:18,312 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/1a6cbc9f12744dbcbe4c3c2688d3aad7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/1a6cbc9f12744dbcbe4c3c2688d3aad7 2024-11-07T14:15:18,319 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/61d165453f4e41d1af78233d08ff6f6b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/61d165453f4e41d1af78233d08ff6f6b 2024-11-07T14:15:18,321 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/fd4a86f614e44bb185d870632a9919ec to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/fd4a86f614e44bb185d870632a9919ec 2024-11-07T14:15:18,322 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2ab499e05a774537bb68b34ad2a84aec to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2ab499e05a774537bb68b34ad2a84aec 2024-11-07T14:15:18,323 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/784f60d52ffc4dc78281072c23e27b37 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/784f60d52ffc4dc78281072c23e27b37 2024-11-07T14:15:18,324 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/aa6e27e6273a4e54be10fbb0fa3d741c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/aa6e27e6273a4e54be10fbb0fa3d741c 2024-11-07T14:15:18,325 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8662e79f6d914ef4b140fa4f4c6eedba to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8662e79f6d914ef4b140fa4f4c6eedba 2024-11-07T14:15:18,327 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/332af80fa04e4633804e88a58fe28b6b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/332af80fa04e4633804e88a58fe28b6b 2024-11-07T14:15:18,328 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e1514b23cca74281b46e3d712e3a40d2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e1514b23cca74281b46e3d712e3a40d2 2024-11-07T14:15:18,329 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8fda8b072ed54087876043c8b3ce905d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/8fda8b072ed54087876043c8b3ce905d 2024-11-07T14:15:18,331 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/713f410b216c46f1a66f23bad7813e32 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/713f410b216c46f1a66f23bad7813e32 2024-11-07T14:15:18,332 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2f069f50beee42b4b7479452d89e0b3e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/2f069f50beee42b4b7479452d89e0b3e 2024-11-07T14:15:18,333 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/6e3dc1acff86419d956cb4869a250dfc to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/6e3dc1acff86419d956cb4869a250dfc 2024-11-07T14:15:18,334 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/ca82634f16cf4abb8c0847acb1e8c1c0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/ca82634f16cf4abb8c0847acb1e8c1c0 2024-11-07T14:15:18,335 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/c4168668c03243c0823af4fc9e36c55d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/c4168668c03243c0823af4fc9e36c55d 2024-11-07T14:15:18,336 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/12d6e4994e704211859ca02488b37055 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/12d6e4994e704211859ca02488b37055 2024-11-07T14:15:18,337 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/59e3365cf8b24fe8aa30035223a164e1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/59e3365cf8b24fe8aa30035223a164e1 2024-11-07T14:15:18,338 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e82eeab45be3433584dfdfe61314dcfd to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/e82eeab45be3433584dfdfe61314dcfd 2024-11-07T14:15:18,339 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0b000c2124e47d9800731bc2f38cb59 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0b000c2124e47d9800731bc2f38cb59 2024-11-07T14:15:18,341 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/a8d768b77644438eb6a8912d7d8b5670 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/a8d768b77644438eb6a8912d7d8b5670 2024-11-07T14:15:18,343 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d9e08a1cc8aa45eb95d663982b204881 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d9e08a1cc8aa45eb95d663982b204881 2024-11-07T14:15:18,345 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/83efd4ff7f65480595b566fc3741b5de to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/83efd4ff7f65480595b566fc3741b5de 2024-11-07T14:15:18,346 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/dda6c659153f496a94f7b78042bfba1e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/dda6c659153f496a94f7b78042bfba1e 2024-11-07T14:15:18,347 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/84be3457045e428ea2bfd6e436d2146d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/84be3457045e428ea2bfd6e436d2146d 2024-11-07T14:15:18,349 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/72c9c75e285349cf9dbaef814fc324c9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/72c9c75e285349cf9dbaef814fc324c9 2024-11-07T14:15:18,350 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/9e4c633db7a0402db7726b8ee12b3424 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/9e4c633db7a0402db7726b8ee12b3424 2024-11-07T14:15:18,351 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/0e8e5a8be6d1457188c801d37c0eb8a0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/0e8e5a8be6d1457188c801d37c0eb8a0 2024-11-07T14:15:18,353 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/35f02f955d6249d2b9c8eba99196cdbe to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/35f02f955d6249d2b9c8eba99196cdbe 2024-11-07T14:15:18,354 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/41679067d583450ebcdb872ac2d22a40 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/41679067d583450ebcdb872ac2d22a40 2024-11-07T14:15:18,355 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/95d5f97230ae42c784443ec3f08e1341 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/95d5f97230ae42c784443ec3f08e1341 2024-11-07T14:15:18,356 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0a0fa226b624ce693d3bd88b4bd1c88 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d0a0fa226b624ce693d3bd88b4bd1c88 2024-11-07T14:15:18,357 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d782e4f821974d398582ec555c035831 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/d782e4f821974d398582ec555c035831 2024-11-07T14:15:18,358 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f6ecc581930a4f5ba196e2a892d468d8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/f6ecc581930a4f5ba196e2a892d468d8 2024-11-07T14:15:18,360 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/08fdbc73454f4c9288750bdd3fdb0a02, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/bfb4a33449a04ed691fdcc4d7e3dda8f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/ba14eece9cab4046bf02adb556109deb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/71a4d7a5326f4232a64fbf3914b44fd2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/f7cf9ab8004f4be8be5f0b1828d83087, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/20bc14f2616146eab50d670afdc30def, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e440785361f14fb5b390f37ea16c6a99, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9137498bbcaa4cf6a0279b7ccd4af88e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8fb12b0c1b984f0e976ed5e6aa2b2cf2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/5c7bda094443439d86d131aca449cf20, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/d2197c08dcac4ea9b27451f704be8656, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63d5c244c4944b7d8021611a71d417b3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/b7d32a682af14c95976fbc9857d10c35, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/fb688aaee4ed4d9a867513420694862d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/31b5778a430a4ad2a5857389cf27d8e3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/943e615800f247fc9ef7231011ff2d54, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/57ff16f3362247bcae282d11186062a1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/032708732e374a899f56630790056c6e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7019a6e3be53431c96113760bfdb5014, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/18f5dfacf35c4dde8995330b9b5cf96a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/214575fec3114c97b552e268e4699030, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/25b7bcbcb6ad452b878bfb55e21e9bab, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7877cf9d71194ad696911a22bf42990f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/4496ecdd743845ed8e289aca599c9958, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/0ea81ef433f245f5a8312d27fb9d9141, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7c0c8f766133408f8cf575920a1ef89d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/df90592e805a4278b6395e2df1d30a58, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e2a3348634604c558fe4ae5271427fd2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/16866c29e9ac4141973c64dd65c16890, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/935fcba155bb40c1bf9674c1e12e1c92, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63c5aa64d4ef4971aa5e59f85559e215, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/729dc97e362e44bca83e18aa259dd3e1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7936869fba7944faad9c64fc7f5c4529, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9c4033b37adb43a2b3f45759ea5e8826, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8a1be277715a42489b0e1fdc2e26e090, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/161ee585bb7c400ab8570512feafe038] to archive 2024-11-07T14:15:18,361 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:15:18,362 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/08fdbc73454f4c9288750bdd3fdb0a02 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/08fdbc73454f4c9288750bdd3fdb0a02 2024-11-07T14:15:18,363 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/bfb4a33449a04ed691fdcc4d7e3dda8f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/bfb4a33449a04ed691fdcc4d7e3dda8f 2024-11-07T14:15:18,364 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/ba14eece9cab4046bf02adb556109deb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/ba14eece9cab4046bf02adb556109deb 2024-11-07T14:15:18,364 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/71a4d7a5326f4232a64fbf3914b44fd2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/71a4d7a5326f4232a64fbf3914b44fd2 2024-11-07T14:15:18,365 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/f7cf9ab8004f4be8be5f0b1828d83087 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/f7cf9ab8004f4be8be5f0b1828d83087 2024-11-07T14:15:18,366 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/20bc14f2616146eab50d670afdc30def to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/20bc14f2616146eab50d670afdc30def 2024-11-07T14:15:18,367 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e440785361f14fb5b390f37ea16c6a99 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e440785361f14fb5b390f37ea16c6a99 2024-11-07T14:15:18,368 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9137498bbcaa4cf6a0279b7ccd4af88e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9137498bbcaa4cf6a0279b7ccd4af88e 2024-11-07T14:15:18,369 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8fb12b0c1b984f0e976ed5e6aa2b2cf2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8fb12b0c1b984f0e976ed5e6aa2b2cf2 2024-11-07T14:15:18,369 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/5c7bda094443439d86d131aca449cf20 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/5c7bda094443439d86d131aca449cf20 2024-11-07T14:15:18,370 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/d2197c08dcac4ea9b27451f704be8656 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/d2197c08dcac4ea9b27451f704be8656 2024-11-07T14:15:18,371 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63d5c244c4944b7d8021611a71d417b3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63d5c244c4944b7d8021611a71d417b3 2024-11-07T14:15:18,373 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/b7d32a682af14c95976fbc9857d10c35 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/b7d32a682af14c95976fbc9857d10c35 2024-11-07T14:15:18,374 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/fb688aaee4ed4d9a867513420694862d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/fb688aaee4ed4d9a867513420694862d 2024-11-07T14:15:18,375 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/31b5778a430a4ad2a5857389cf27d8e3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/31b5778a430a4ad2a5857389cf27d8e3 2024-11-07T14:15:18,376 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/943e615800f247fc9ef7231011ff2d54 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/943e615800f247fc9ef7231011ff2d54 2024-11-07T14:15:18,377 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/57ff16f3362247bcae282d11186062a1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/57ff16f3362247bcae282d11186062a1 2024-11-07T14:15:18,378 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/032708732e374a899f56630790056c6e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/032708732e374a899f56630790056c6e 2024-11-07T14:15:18,378 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7019a6e3be53431c96113760bfdb5014 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7019a6e3be53431c96113760bfdb5014 2024-11-07T14:15:18,379 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/18f5dfacf35c4dde8995330b9b5cf96a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/18f5dfacf35c4dde8995330b9b5cf96a 2024-11-07T14:15:18,380 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/214575fec3114c97b552e268e4699030 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/214575fec3114c97b552e268e4699030 2024-11-07T14:15:18,381 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/25b7bcbcb6ad452b878bfb55e21e9bab to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/25b7bcbcb6ad452b878bfb55e21e9bab 2024-11-07T14:15:18,382 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7877cf9d71194ad696911a22bf42990f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7877cf9d71194ad696911a22bf42990f 2024-11-07T14:15:18,383 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/4496ecdd743845ed8e289aca599c9958 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/4496ecdd743845ed8e289aca599c9958 2024-11-07T14:15:18,384 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/0ea81ef433f245f5a8312d27fb9d9141 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/0ea81ef433f245f5a8312d27fb9d9141 2024-11-07T14:15:18,384 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7c0c8f766133408f8cf575920a1ef89d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7c0c8f766133408f8cf575920a1ef89d 2024-11-07T14:15:18,385 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/df90592e805a4278b6395e2df1d30a58 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/df90592e805a4278b6395e2df1d30a58 2024-11-07T14:15:18,386 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e2a3348634604c558fe4ae5271427fd2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/e2a3348634604c558fe4ae5271427fd2 2024-11-07T14:15:18,387 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/16866c29e9ac4141973c64dd65c16890 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/16866c29e9ac4141973c64dd65c16890 2024-11-07T14:15:18,388 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/935fcba155bb40c1bf9674c1e12e1c92 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/935fcba155bb40c1bf9674c1e12e1c92 2024-11-07T14:15:18,389 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63c5aa64d4ef4971aa5e59f85559e215 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/63c5aa64d4ef4971aa5e59f85559e215 2024-11-07T14:15:18,390 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/729dc97e362e44bca83e18aa259dd3e1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/729dc97e362e44bca83e18aa259dd3e1 2024-11-07T14:15:18,390 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7936869fba7944faad9c64fc7f5c4529 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/7936869fba7944faad9c64fc7f5c4529 2024-11-07T14:15:18,391 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9c4033b37adb43a2b3f45759ea5e8826 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/9c4033b37adb43a2b3f45759ea5e8826 2024-11-07T14:15:18,392 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8a1be277715a42489b0e1fdc2e26e090 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/8a1be277715a42489b0e1fdc2e26e090 2024-11-07T14:15:18,393 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/161ee585bb7c400ab8570512feafe038 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/161ee585bb7c400ab8570512feafe038 2024-11-07T14:15:18,394 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9018ad9ecd474d17a261d274a0517feb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d80c883b5e214e019c7c77431ba5908d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f3999e84104c4c8fa48e79153004b13f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2ae168f354c146d998f7d247a8d37b47, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4db46817319b4f118ae32f0e81f8bc32, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e308f0874da34cc1bbc5eaffaf8f8767, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/47622c4b84f8469988d508b62a96909e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8cdcfa97284438b98d95cd10226d85f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/26b250f8523a4c1f87b26ac0530ed63a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4c48ac5bf3414103bd35aca06d592606, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f0a46c1183324eb68780a81be00ca770, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f9ca0aaf3994b3ba4f10462ead14109, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f321ee3facbd4d2faf442332f92a758f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8bd56f024f1446198fce6b8fb39efa2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d66b1b6afff1438eb2ce375ff81b9289, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d52c58079c7d4ee99a403d112977a285, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e2d561d17c634d6fa3211aa8bf0f541d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e841f56199b5498fa339ba2504b6d446, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/0dd6f89f19cd4425be0a347f32473ebd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/aa5bc85072ab4d8c88b29e4aff85015d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f04105c6cb8444296aa7caefb7a71f3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/526ff8cc4ba74e8b8bd3f21facc4c2ae, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/70f2116436e448809c3e0a5d060d5224, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c5d71568f93446bd8cd76c04c13805e1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/de8e2b9eaa094b498c81c448948d04b0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/044031b9e07c4e71aa83fb7c51d97a21, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/30ba41c3240748bcbcfbb6490f59048a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f9fdda16be6c470dbc69f58ad66dbdb2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/3b1aeb98f0fd49418e86804937015fff, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/7e8182133b484b0a9f37aa5dc8a6bc20, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/77267a6b29ed44859c2302d0f2b2c9be, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4ce522a593ea47ae9bd3fea94859ca3b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/ec9c9bf84e4f44889e372096f11e7337, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9907b2d1ae574345a3ea0e095824cc51, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2029ace6d64d4e60b317318058f28a34] to archive 2024-11-07T14:15:18,395 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:15:18,396 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9018ad9ecd474d17a261d274a0517feb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9018ad9ecd474d17a261d274a0517feb 2024-11-07T14:15:18,397 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d80c883b5e214e019c7c77431ba5908d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d80c883b5e214e019c7c77431ba5908d 2024-11-07T14:15:18,398 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f3999e84104c4c8fa48e79153004b13f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f3999e84104c4c8fa48e79153004b13f 2024-11-07T14:15:18,399 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2ae168f354c146d998f7d247a8d37b47 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2ae168f354c146d998f7d247a8d37b47 2024-11-07T14:15:18,400 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4db46817319b4f118ae32f0e81f8bc32 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4db46817319b4f118ae32f0e81f8bc32 2024-11-07T14:15:18,400 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e308f0874da34cc1bbc5eaffaf8f8767 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e308f0874da34cc1bbc5eaffaf8f8767 2024-11-07T14:15:18,401 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/47622c4b84f8469988d508b62a96909e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/47622c4b84f8469988d508b62a96909e 2024-11-07T14:15:18,402 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8cdcfa97284438b98d95cd10226d85f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8cdcfa97284438b98d95cd10226d85f 2024-11-07T14:15:18,403 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/26b250f8523a4c1f87b26ac0530ed63a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/26b250f8523a4c1f87b26ac0530ed63a 2024-11-07T14:15:18,404 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4c48ac5bf3414103bd35aca06d592606 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4c48ac5bf3414103bd35aca06d592606 2024-11-07T14:15:18,404 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f0a46c1183324eb68780a81be00ca770 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f0a46c1183324eb68780a81be00ca770 2024-11-07T14:15:18,406 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f9ca0aaf3994b3ba4f10462ead14109 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f9ca0aaf3994b3ba4f10462ead14109 2024-11-07T14:15:18,407 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f321ee3facbd4d2faf442332f92a758f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f321ee3facbd4d2faf442332f92a758f 2024-11-07T14:15:18,408 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8bd56f024f1446198fce6b8fb39efa2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c8bd56f024f1446198fce6b8fb39efa2 2024-11-07T14:15:18,409 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d66b1b6afff1438eb2ce375ff81b9289 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d66b1b6afff1438eb2ce375ff81b9289 2024-11-07T14:15:18,410 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d52c58079c7d4ee99a403d112977a285 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/d52c58079c7d4ee99a403d112977a285 2024-11-07T14:15:18,411 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e2d561d17c634d6fa3211aa8bf0f541d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e2d561d17c634d6fa3211aa8bf0f541d 2024-11-07T14:15:18,412 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e841f56199b5498fa339ba2504b6d446 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/e841f56199b5498fa339ba2504b6d446 2024-11-07T14:15:18,413 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/0dd6f89f19cd4425be0a347f32473ebd to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/0dd6f89f19cd4425be0a347f32473ebd 2024-11-07T14:15:18,413 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/aa5bc85072ab4d8c88b29e4aff85015d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/aa5bc85072ab4d8c88b29e4aff85015d 2024-11-07T14:15:18,414 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f04105c6cb8444296aa7caefb7a71f3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4f04105c6cb8444296aa7caefb7a71f3 2024-11-07T14:15:18,415 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/526ff8cc4ba74e8b8bd3f21facc4c2ae to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/526ff8cc4ba74e8b8bd3f21facc4c2ae 2024-11-07T14:15:18,416 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/70f2116436e448809c3e0a5d060d5224 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/70f2116436e448809c3e0a5d060d5224 2024-11-07T14:15:18,417 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c5d71568f93446bd8cd76c04c13805e1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/c5d71568f93446bd8cd76c04c13805e1 2024-11-07T14:15:18,418 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/de8e2b9eaa094b498c81c448948d04b0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/de8e2b9eaa094b498c81c448948d04b0 2024-11-07T14:15:18,418 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/044031b9e07c4e71aa83fb7c51d97a21 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/044031b9e07c4e71aa83fb7c51d97a21 2024-11-07T14:15:18,419 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/30ba41c3240748bcbcfbb6490f59048a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/30ba41c3240748bcbcfbb6490f59048a 2024-11-07T14:15:18,420 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f9fdda16be6c470dbc69f58ad66dbdb2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/f9fdda16be6c470dbc69f58ad66dbdb2 2024-11-07T14:15:18,421 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/3b1aeb98f0fd49418e86804937015fff to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/3b1aeb98f0fd49418e86804937015fff 2024-11-07T14:15:18,421 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/7e8182133b484b0a9f37aa5dc8a6bc20 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/7e8182133b484b0a9f37aa5dc8a6bc20 2024-11-07T14:15:18,422 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/77267a6b29ed44859c2302d0f2b2c9be to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/77267a6b29ed44859c2302d0f2b2c9be 2024-11-07T14:15:18,423 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4ce522a593ea47ae9bd3fea94859ca3b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/4ce522a593ea47ae9bd3fea94859ca3b 2024-11-07T14:15:18,423 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/ec9c9bf84e4f44889e372096f11e7337 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/ec9c9bf84e4f44889e372096f11e7337 2024-11-07T14:15:18,424 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9907b2d1ae574345a3ea0e095824cc51 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/9907b2d1ae574345a3ea0e095824cc51 2024-11-07T14:15:18,425 DEBUG [StoreCloser-TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2029ace6d64d4e60b317318058f28a34 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/2029ace6d64d4e60b317318058f28a34 2024-11-07T14:15:18,429 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/recovered.edits/537.seqid, newMaxSeqId=537, maxSeqId=1 2024-11-07T14:15:18,429 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. 2024-11-07T14:15:18,429 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 1d800a645cbbc59d7375bdd5295bbe51: Waiting for close lock at 1730988917055Running coprocessor pre-close hooks at 1730988917055Disabling compacts and flushes for region at 1730988917055Disabling writes for close at 1730988917055Obtaining lock to block concurrent updates at 1730988917055Preparing flush snapshotting stores in 1d800a645cbbc59d7375bdd5295bbe51 at 1730988917055Finished memstore snapshotting TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730988917056 (+1 ms)Flushing stores of TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51. at 1730988917056Flushing 1d800a645cbbc59d7375bdd5295bbe51/A: creating writer at 1730988917056Flushing 1d800a645cbbc59d7375bdd5295bbe51/A: appending metadata at 1730988917058 (+2 ms)Flushing 1d800a645cbbc59d7375bdd5295bbe51/A: closing flushed file at 1730988917058Flushing 1d800a645cbbc59d7375bdd5295bbe51/B: creating writer at 1730988917466 (+408 ms)Flushing 1d800a645cbbc59d7375bdd5295bbe51/B: appending metadata at 1730988917468 (+2 ms)Flushing 1d800a645cbbc59d7375bdd5295bbe51/B: closing flushed file at 1730988917468Flushing 1d800a645cbbc59d7375bdd5295bbe51/C: creating writer at 1730988917877 (+409 ms)Flushing 1d800a645cbbc59d7375bdd5295bbe51/C: appending metadata at 1730988917880 (+3 ms)Flushing 1d800a645cbbc59d7375bdd5295bbe51/C: closing flushed file at 1730988917880Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b64163a: reopening flushed file at 1730988918288 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3738dd72: reopening flushed file at 1730988918292 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63304e0d: reopening flushed file at 1730988918298 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1d800a645cbbc59d7375bdd5295bbe51 in 1250ms, sequenceid=534, compaction requested=true at 1730988918305 (+7 ms)Writing region close event to WAL at 1730988918426 (+121 ms)Running coprocessor post-close hooks at 1730988918429 (+3 ms)Closed at 1730988918429 2024-11-07T14:15:18,432 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:18,432 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=1d800a645cbbc59d7375bdd5295bbe51, regionState=CLOSED 2024-11-07T14:15:18,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:15:18,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-07T14:15:18,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d800a645cbbc59d7375bdd5295bbe51, server=b7cc9d764de6,37737,1730988826985 in 1.5330 sec 2024-11-07T14:15:18,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-07T14:15:18,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d800a645cbbc59d7375bdd5295bbe51, UNASSIGN in 1.5370 sec 2024-11-07T14:15:18,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T14:15:18,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5410 sec 2024-11-07T14:15:18,441 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988918441"}]},"ts":"1730988918441"} 2024-11-07T14:15:18,443 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T14:15:18,443 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T14:15:18,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5500 sec 2024-11-07T14:15:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T14:15:19,042 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:19,043 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T14:15:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:19,044 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:19,045 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T14:15:19,047 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:19,051 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/recovered.edits] 2024-11-07T14:15:19,058 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/23688201341b4226bf1634990f2a7fce to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/23688201341b4226bf1634990f2a7fce 2024-11-07T14:15:19,066 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4bb9f1e2a25d4184a31ddc1ec45de7ff to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/4bb9f1e2a25d4184a31ddc1ec45de7ff 2024-11-07T14:15:19,072 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/983bfd3d40644aff9bc50206be1d99f6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/A/983bfd3d40644aff9bc50206be1d99f6 2024-11-07T14:15:19,074 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/192ed9e4f5534fed97c678a29cb9d78a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/192ed9e4f5534fed97c678a29cb9d78a 2024-11-07T14:15:19,075 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/26711f73590d4dc5ba4b9e21fdddef9e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/26711f73590d4dc5ba4b9e21fdddef9e 2024-11-07T14:15:19,076 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/a5863bc0868a47048c203db748bd3d6d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/B/a5863bc0868a47048c203db748bd3d6d 2024-11-07T14:15:19,091 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/012e775888ce4e6d9d4e54acfd6d3f78 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/012e775888ce4e6d9d4e54acfd6d3f78 2024-11-07T14:15:19,098 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/21c200560c4c41ef8b64ca8de1db55df to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/21c200560c4c41ef8b64ca8de1db55df 2024-11-07T14:15:19,107 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/45bbb0a5098f43b09e1b4273d69453cf to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/C/45bbb0a5098f43b09e1b4273d69453cf 2024-11-07T14:15:19,117 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/recovered.edits/537.seqid to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51/recovered.edits/537.seqid 2024-11-07T14:15:19,117 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/1d800a645cbbc59d7375bdd5295bbe51 2024-11-07T14:15:19,118 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T14:15:19,120 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:19,124 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T14:15:19,126 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T14:15:19,128 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:19,128 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T14:15:19,128 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730988919128"}]},"ts":"9223372036854775807"} 2024-11-07T14:15:19,130 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T14:15:19,130 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1d800a645cbbc59d7375bdd5295bbe51, NAME => 'TestAcidGuarantees,,1730988891665.1d800a645cbbc59d7375bdd5295bbe51.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T14:15:19,130 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T14:15:19,131 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730988919130"}]},"ts":"9223372036854775807"} 2024-11-07T14:15:19,133 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T14:15:19,134 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:19,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-07T14:15:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T14:15:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T14:15:19,156 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:19,164 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 241), OpenFileDescriptor=455 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=563 (was 553) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=239 (was 1136) 2024-11-07T14:15:19,172 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=563, ProcessCount=11, AvailableMemoryMB=239 2024-11-07T14:15:19,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:15:19,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:15:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:19,176 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:15:19,176 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:19,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-07T14:15:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T14:15:19,177 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:15:19,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742169_1345 (size=959) 2024-11-07T14:15:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T14:15:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T14:15:19,586 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 2024-11-07T14:15:19,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742170_1346 (size=53) 2024-11-07T14:15:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T14:15:19,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:15:19,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 186148ebfb7e3f0985eb6829e86c465f, disabling compactions & flushes 2024-11-07T14:15:19,993 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:19,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:19,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. after waiting 0 ms 2024-11-07T14:15:19,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:19,993 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:19,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 186148ebfb7e3f0985eb6829e86c465f: Waiting for close lock at 1730988919993Disabling compacts and flushes for region at 1730988919993Disabling writes for close at 1730988919993Writing region close event to WAL at 1730988919993Closed at 1730988919993 2024-11-07T14:15:19,994 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:15:19,995 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730988919994"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730988919994"}]},"ts":"1730988919994"} 2024-11-07T14:15:19,997 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T14:15:19,998 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:15:19,998 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988919998"}]},"ts":"1730988919998"} 2024-11-07T14:15:20,000 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T14:15:20,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, ASSIGN}] 2024-11-07T14:15:20,002 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, ASSIGN 2024-11-07T14:15:20,003 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, ASSIGN; state=OFFLINE, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=false 2024-11-07T14:15:20,154 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=186148ebfb7e3f0985eb6829e86c465f, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:20,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, ASSIGN because future has completed 2024-11-07T14:15:20,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:15:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T14:15:20,314 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:20,314 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:15:20,315 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,315 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:15:20,316 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,316 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,319 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,322 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 186148ebfb7e3f0985eb6829e86c465f columnFamilyName A 2024-11-07T14:15:20,322 DEBUG [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:20,323 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(327): Store=186148ebfb7e3f0985eb6829e86c465f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:20,323 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,325 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 186148ebfb7e3f0985eb6829e86c465f columnFamilyName B 2024-11-07T14:15:20,325 DEBUG [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:20,326 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(327): Store=186148ebfb7e3f0985eb6829e86c465f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:20,326 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,327 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 186148ebfb7e3f0985eb6829e86c465f columnFamilyName C 2024-11-07T14:15:20,328 DEBUG [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:20,328 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(327): Store=186148ebfb7e3f0985eb6829e86c465f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:20,328 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,329 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,330 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,331 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,332 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,332 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:15:20,333 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,336 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:15:20,336 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened 186148ebfb7e3f0985eb6829e86c465f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69710402, jitterRate=0.038765937089920044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:15:20,336 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:20,337 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for 186148ebfb7e3f0985eb6829e86c465f: Running coprocessor pre-open hook at 1730988920316Writing region info on filesystem at 1730988920316Initializing all the Stores at 1730988920318 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988920318Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988920318Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988920318Cleaning up temporary data from old regions at 1730988920332 (+14 ms)Running coprocessor post-open hooks at 1730988920336 (+4 ms)Region opened successfully at 1730988920337 (+1 ms) 2024-11-07T14:15:20,338 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., pid=97, masterSystemTime=1730988920307 2024-11-07T14:15:20,341 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:20,341 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:20,341 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=186148ebfb7e3f0985eb6829e86c465f, regionState=OPEN, openSeqNum=2, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:20,343 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40745 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=b7cc9d764de6,37737,1730988826985, table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T14:15:20,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:15:20,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-07T14:15:20,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 in 189 msec 2024-11-07T14:15:20,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-07T14:15:20,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, ASSIGN in 347 msec 2024-11-07T14:15:20,351 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:15:20,351 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988920351"}]},"ts":"1730988920351"} 2024-11-07T14:15:20,353 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T14:15:20,354 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:15:20,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-07T14:15:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T14:15:21,333 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:21,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29820261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:21,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:21,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:21,335 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a58596, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:21,337 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67fb9f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:21,338 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:21,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:21,341 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T14:15:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b7cc9d764de6,40745,1730988826115 2024-11-07T14:15:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7902dc6f 2024-11-07T14:15:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T14:15:21,342 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T14:15:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:15:21,344 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:15:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:21,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742171_1347 (size=995) 2024-11-07T14:15:21,756 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T14:15:21,756 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T14:15:21,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:15:21,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, REOPEN/MOVE}] 2024-11-07T14:15:21,762 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, REOPEN/MOVE 2024-11-07T14:15:21,763 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=186148ebfb7e3f0985eb6829e86c465f, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:21,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, REOPEN/MOVE because future has completed 2024-11-07T14:15:21,764 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:15:21,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:15:21,917 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:21,917 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:15:21,917 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing 186148ebfb7e3f0985eb6829e86c465f, disabling compactions & flushes 2024-11-07T14:15:21,917 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:21,917 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:21,917 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. after waiting 0 ms 2024-11-07T14:15:21,917 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:21,921 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T14:15:21,922 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:21,922 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for 186148ebfb7e3f0985eb6829e86c465f: Waiting for close lock at 1730988921917Running coprocessor pre-close hooks at 1730988921917Disabling compacts and flushes for region at 1730988921917Disabling writes for close at 1730988921917Writing region close event to WAL at 1730988921918 (+1 ms)Running coprocessor post-close hooks at 1730988921922 (+4 ms)Closed at 1730988921922 2024-11-07T14:15:21,922 WARN [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: 186148ebfb7e3f0985eb6829e86c465f to self. 2024-11-07T14:15:21,924 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:21,925 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=186148ebfb7e3f0985eb6829e86c465f, regionState=CLOSED 2024-11-07T14:15:21,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:15:21,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-07T14:15:21,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 in 167 msec 2024-11-07T14:15:21,934 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, REOPEN/MOVE; state=CLOSED, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=true 2024-11-07T14:15:22,084 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=186148ebfb7e3f0985eb6829e86c465f, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, REOPEN/MOVE because future has completed 2024-11-07T14:15:22,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:15:22,241 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,242 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:15:22,242 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,242 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:15:22,242 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,242 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,243 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,244 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 186148ebfb7e3f0985eb6829e86c465f columnFamilyName A 2024-11-07T14:15:22,245 DEBUG [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:22,246 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(327): Store=186148ebfb7e3f0985eb6829e86c465f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:22,246 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,247 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 186148ebfb7e3f0985eb6829e86c465f columnFamilyName B 2024-11-07T14:15:22,247 DEBUG [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:22,247 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(327): Store=186148ebfb7e3f0985eb6829e86c465f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:22,248 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,248 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 186148ebfb7e3f0985eb6829e86c465f columnFamilyName C 2024-11-07T14:15:22,248 DEBUG [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:22,249 INFO [StoreOpener-186148ebfb7e3f0985eb6829e86c465f-1 {}] regionserver.HStore(327): Store=186148ebfb7e3f0985eb6829e86c465f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:22,249 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,250 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,251 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,252 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,252 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,252 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:15:22,253 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,254 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened 186148ebfb7e3f0985eb6829e86c465f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71291890, jitterRate=0.062331944704055786}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:15:22,254 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,255 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for 186148ebfb7e3f0985eb6829e86c465f: Running coprocessor pre-open hook at 1730988922242Writing region info on filesystem at 1730988922242Initializing all the Stores at 1730988922243 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988922243Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988922243Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988922243Cleaning up temporary data from old regions at 1730988922252 (+9 ms)Running coprocessor post-open hooks at 1730988922254 (+2 ms)Region opened successfully at 1730988922255 (+1 ms) 2024-11-07T14:15:22,255 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., pid=102, masterSystemTime=1730988922238 2024-11-07T14:15:22,257 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,257 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,258 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=186148ebfb7e3f0985eb6829e86c465f, regionState=OPEN, openSeqNum=5, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:15:22,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-07T14:15:22,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 in 174 msec 2024-11-07T14:15:22,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-07T14:15:22,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, REOPEN/MOVE in 500 msec 2024-11-07T14:15:22,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-07T14:15:22,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-07T14:15:22,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-11-07T14:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-07T14:15:22,268 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:22,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c07883b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,270 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15df9d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,272 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fc5064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d62a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,276 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a28387, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,278 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364937b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6513d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45588246, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47940271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,285 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a6929a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,287 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e74ab4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,290 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419eac72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,291 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2b532b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469d248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,294 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39be5fd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,295 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@774a8c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8c61ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f19a01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314e85aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147a1007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,305 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c96be44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ffc8e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4d201d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,309 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5d0926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,311 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a5dd5f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2b45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:22,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:22,314 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f85f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:22,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fabee92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:22,319 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,322 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,322 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,323 DEBUG [Thread-1581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,323 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,326 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,328 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,328 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,328 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,329 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:22,334 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:15:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-07T14:15:22,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T14:15:22,343 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:22,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,349 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:22,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:15:22,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a9cb715e4e1344db8e4e90f5048389f7_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988922348/Put/seqid=0 2024-11-07T14:15:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988982405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988982414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988982420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988982420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988982422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742172_1348 (size=12154) 2024-11-07T14:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T14:15:22,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988982533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988982535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988982543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988982543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988982543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T14:15:22,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988982744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988982747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988982757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988982757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988982757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:22,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:22,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,853 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:22,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a9cb715e4e1344db8e4e90f5048389f7_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a9cb715e4e1344db8e4e90f5048389f7_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:22,865 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/30dae00288e7474796992a0624693e55, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:22,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/30dae00288e7474796992a0624693e55 is 175, key is test_row_0/A:col10/1730988922348/Put/seqid=0 2024-11-07T14:15:22,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742173_1349 (size=30955) 2024-11-07T14:15:22,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/30dae00288e7474796992a0624693e55 2024-11-07T14:15:22,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/72bde7a0cbb94adf9f1c845ac51748f6 is 50, key is test_row_0/B:col10/1730988922348/Put/seqid=0 2024-11-07T14:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T14:15:22,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:22,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:22,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742174_1350 (size=12001) 2024-11-07T14:15:23,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988983062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988983062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988983072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988983083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988983091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:23,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/72bde7a0cbb94adf9f1c845ac51748f6 2024-11-07T14:15:23,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/9f5ffe5c242b4958bbe0960a493e07cd is 50, key is test_row_0/C:col10/1730988922348/Put/seqid=0 2024-11-07T14:15:23,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T14:15:23,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742175_1351 (size=12001) 2024-11-07T14:15:23,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988983576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988983582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988983582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988983593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988983603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:23,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:23,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:23,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:23,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/9f5ffe5c242b4958bbe0960a493e07cd 2024-11-07T14:15:23,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/30dae00288e7474796992a0624693e55 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/30dae00288e7474796992a0624693e55 2024-11-07T14:15:23,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:23,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:23,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:23,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:23,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/30dae00288e7474796992a0624693e55, entries=150, sequenceid=16, filesize=30.2 K 2024-11-07T14:15:23,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/72bde7a0cbb94adf9f1c845ac51748f6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72bde7a0cbb94adf9f1c845ac51748f6 2024-11-07T14:15:23,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72bde7a0cbb94adf9f1c845ac51748f6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T14:15:23,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/9f5ffe5c242b4958bbe0960a493e07cd as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9f5ffe5c242b4958bbe0960a493e07cd 2024-11-07T14:15:23,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9f5ffe5c242b4958bbe0960a493e07cd, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T14:15:23,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 186148ebfb7e3f0985eb6829e86c465f in 1613ms, sequenceid=16, compaction requested=false 2024-11-07T14:15:23,962 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T14:15:23,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:24,002 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:15:24,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T14:15:24,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:24,083 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:15:24,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078f5341915996417088934c4e346f418a_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988922403/Put/seqid=0 2024-11-07T14:15:24,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742176_1352 (size=12154) 2024-11-07T14:15:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T14:15:24,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:24,523 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078f5341915996417088934c4e346f418a_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078f5341915996417088934c4e346f418a_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:24,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/83b1575ed6174c3395554cd4a5df2edf, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:24,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/83b1575ed6174c3395554cd4a5df2edf is 175, key is test_row_0/A:col10/1730988922403/Put/seqid=0 2024-11-07T14:15:24,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742177_1353 (size=30955) 2024-11-07T14:15:24,567 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/83b1575ed6174c3395554cd4a5df2edf 2024-11-07T14:15:24,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c845ffecff0544b18e07f7aa6d096df7 is 50, key is test_row_0/B:col10/1730988922403/Put/seqid=0 2024-11-07T14:15:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:24,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:24,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742178_1354 (size=12001) 2024-11-07T14:15:24,632 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c845ffecff0544b18e07f7aa6d096df7 2024-11-07T14:15:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/43db3ec5401f467aac712eb6de551eaa is 50, key is test_row_0/C:col10/1730988922403/Put/seqid=0 2024-11-07T14:15:24,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988984631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988984633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742179_1355 (size=12001) 2024-11-07T14:15:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/43db3ec5401f467aac712eb6de551eaa 2024-11-07T14:15:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988984641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/83b1575ed6174c3395554cd4a5df2edf as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/83b1575ed6174c3395554cd4a5df2edf 2024-11-07T14:15:24,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988984645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988984645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,672 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/83b1575ed6174c3395554cd4a5df2edf, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T14:15:24,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c845ffecff0544b18e07f7aa6d096df7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c845ffecff0544b18e07f7aa6d096df7 2024-11-07T14:15:24,679 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c845ffecff0544b18e07f7aa6d096df7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T14:15:24,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/43db3ec5401f467aac712eb6de551eaa as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/43db3ec5401f467aac712eb6de551eaa 2024-11-07T14:15:24,684 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/43db3ec5401f467aac712eb6de551eaa, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T14:15:24,685 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 186148ebfb7e3f0985eb6829e86c465f in 602ms, sequenceid=40, compaction requested=false 2024-11-07T14:15:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-07T14:15:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-07T14:15:24,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-07T14:15:24,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3440 sec 2024-11-07T14:15:24,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3550 sec 2024-11-07T14:15:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:24,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T14:15:24,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073d019db827b147b0ace6a465af3b39ce_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988924622/Put/seqid=0 2024-11-07T14:15:24,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742180_1356 (size=17034) 2024-11-07T14:15:24,771 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:24,778 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073d019db827b147b0ace6a465af3b39ce_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073d019db827b147b0ace6a465af3b39ce_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:24,779 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c877c296e0b844c99fd7a6e1b02960e8, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:24,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c877c296e0b844c99fd7a6e1b02960e8 is 175, key is test_row_0/A:col10/1730988924622/Put/seqid=0 2024-11-07T14:15:24,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742181_1357 (size=48139) 2024-11-07T14:15:24,791 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c877c296e0b844c99fd7a6e1b02960e8 2024-11-07T14:15:24,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988984791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988984792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/28fc7ae0d8714e9993fdab75fcc58d9b is 50, key is test_row_0/B:col10/1730988924622/Put/seqid=0 2024-11-07T14:15:24,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988984794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988984795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988984800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742182_1358 (size=12001) 2024-11-07T14:15:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988984902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988984913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988984913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988984913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:24,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988984927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988985122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988985132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988985132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988985135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988985142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/28fc7ae0d8714e9993fdab75fcc58d9b 2024-11-07T14:15:25,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/f2f82fefedb6410fa0f2f5ac3477bcf3 is 50, key is test_row_0/C:col10/1730988924622/Put/seqid=0 2024-11-07T14:15:25,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742183_1359 (size=12001) 2024-11-07T14:15:25,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988985432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988985442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988985442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988985454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988985463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/f2f82fefedb6410fa0f2f5ac3477bcf3 2024-11-07T14:15:25,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c877c296e0b844c99fd7a6e1b02960e8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c877c296e0b844c99fd7a6e1b02960e8 2024-11-07T14:15:25,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c877c296e0b844c99fd7a6e1b02960e8, entries=250, sequenceid=56, filesize=47.0 K 2024-11-07T14:15:25,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/28fc7ae0d8714e9993fdab75fcc58d9b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/28fc7ae0d8714e9993fdab75fcc58d9b 2024-11-07T14:15:25,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/28fc7ae0d8714e9993fdab75fcc58d9b, entries=150, sequenceid=56, filesize=11.7 K 2024-11-07T14:15:25,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/f2f82fefedb6410fa0f2f5ac3477bcf3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/f2f82fefedb6410fa0f2f5ac3477bcf3 2024-11-07T14:15:25,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/f2f82fefedb6410fa0f2f5ac3477bcf3, entries=150, sequenceid=56, filesize=11.7 K 2024-11-07T14:15:25,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 186148ebfb7e3f0985eb6829e86c465f in 916ms, sequenceid=56, compaction requested=true 2024-11-07T14:15:25,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:25,672 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:25,672 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:25,673 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:25,673 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/A is initiating minor compaction (all files) 2024-11-07T14:15:25,673 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/A in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:25,673 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/30dae00288e7474796992a0624693e55, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/83b1575ed6174c3395554cd4a5df2edf, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c877c296e0b844c99fd7a6e1b02960e8] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=107.5 K 2024-11-07T14:15:25,673 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:25,673 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/30dae00288e7474796992a0624693e55, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/83b1575ed6174c3395554cd4a5df2edf, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c877c296e0b844c99fd7a6e1b02960e8] 2024-11-07T14:15:25,673 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:25,673 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/B is initiating minor compaction (all files) 2024-11-07T14:15:25,673 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/B in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:25,673 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72bde7a0cbb94adf9f1c845ac51748f6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c845ffecff0544b18e07f7aa6d096df7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/28fc7ae0d8714e9993fdab75fcc58d9b] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=35.2 K 2024-11-07T14:15:25,673 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 72bde7a0cbb94adf9f1c845ac51748f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730988922333 2024-11-07T14:15:25,673 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30dae00288e7474796992a0624693e55, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730988922333 2024-11-07T14:15:25,674 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c845ffecff0544b18e07f7aa6d096df7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730988922403 2024-11-07T14:15:25,674 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83b1575ed6174c3395554cd4a5df2edf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730988922403 2024-11-07T14:15:25,674 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c877c296e0b844c99fd7a6e1b02960e8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988924622 2024-11-07T14:15:25,674 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 28fc7ae0d8714e9993fdab75fcc58d9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988924622 2024-11-07T14:15:25,678 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:25,684 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#B#compaction#314 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:25,684 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c9685e8ced7d4789b945eb5991c0b2e1 is 50, key is test_row_0/B:col10/1730988924622/Put/seqid=0 2024-11-07T14:15:25,691 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411071eb3a7fd16f44d42badbf9fbf09828ba_186148ebfb7e3f0985eb6829e86c465f store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:25,693 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411071eb3a7fd16f44d42badbf9fbf09828ba_186148ebfb7e3f0985eb6829e86c465f, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:25,693 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071eb3a7fd16f44d42badbf9fbf09828ba_186148ebfb7e3f0985eb6829e86c465f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:25,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742184_1360 (size=12104) 2024-11-07T14:15:25,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742185_1361 (size=4469) 2024-11-07T14:15:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:25,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:25,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T14:15:25,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e90d60bafe19475084c35c4228a23c86_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988925956/Put/seqid=0 2024-11-07T14:15:25,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742186_1362 (size=17034) 2024-11-07T14:15:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988985980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988985981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988985982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988985983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988985990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,103 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c9685e8ced7d4789b945eb5991c0b2e1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c9685e8ced7d4789b945eb5991c0b2e1 2024-11-07T14:15:26,107 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/B of 186148ebfb7e3f0985eb6829e86c465f into c9685e8ced7d4789b945eb5991c0b2e1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:26,107 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:26,107 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/B, priority=13, startTime=1730988925672; duration=0sec 2024-11-07T14:15:26,108 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:26,108 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:B 2024-11-07T14:15:26,108 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:26,108 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:26,108 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/C is initiating minor compaction (all files) 2024-11-07T14:15:26,109 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/C in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:26,109 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9f5ffe5c242b4958bbe0960a493e07cd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/43db3ec5401f467aac712eb6de551eaa, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/f2f82fefedb6410fa0f2f5ac3477bcf3] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=35.2 K 2024-11-07T14:15:26,109 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5ffe5c242b4958bbe0960a493e07cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730988922333 2024-11-07T14:15:26,109 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 43db3ec5401f467aac712eb6de551eaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730988922403 2024-11-07T14:15:26,109 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f82fefedb6410fa0f2f5ac3477bcf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988924622 2024-11-07T14:15:26,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988986102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988986102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988986102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988986102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,114 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:26,114 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/468b57c3566c436281985cc177295fd9 is 175, key is test_row_0/A:col10/1730988924622/Put/seqid=0 2024-11-07T14:15:26,116 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:26,116 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/271bdd73d1f14660adb218583abdec67 is 50, key is test_row_0/C:col10/1730988924622/Put/seqid=0 2024-11-07T14:15:26,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742187_1363 (size=31058) 2024-11-07T14:15:26,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742188_1364 (size=12104) 2024-11-07T14:15:26,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988986112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,126 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/271bdd73d1f14660adb218583abdec67 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/271bdd73d1f14660adb218583abdec67 2024-11-07T14:15:26,136 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/C of 186148ebfb7e3f0985eb6829e86c465f into 271bdd73d1f14660adb218583abdec67(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:26,136 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:26,136 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/C, priority=13, startTime=1730988925672; duration=0sec 2024-11-07T14:15:26,136 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:26,136 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:C 2024-11-07T14:15:26,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988986322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988986322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988986322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988986322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988986332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:26,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e90d60bafe19475084c35c4228a23c86_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e90d60bafe19475084c35c4228a23c86_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:26,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/778a65ff9f604f8b897979b7694ffeb2, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:26,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/778a65ff9f604f8b897979b7694ffeb2 is 175, key is test_row_0/A:col10/1730988925956/Put/seqid=0 2024-11-07T14:15:26,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742189_1365 (size=48139) 2024-11-07T14:15:26,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/778a65ff9f604f8b897979b7694ffeb2 2024-11-07T14:15:26,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/02a0ce8ad6f7425380d7657613b37443 is 50, key is test_row_0/B:col10/1730988925956/Put/seqid=0 2024-11-07T14:15:26,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742190_1366 (size=12001) 2024-11-07T14:15:26,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/02a0ce8ad6f7425380d7657613b37443 2024-11-07T14:15:26,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/d08112df6788493fa98e844cc60213bd is 50, key is test_row_0/C:col10/1730988925956/Put/seqid=0 2024-11-07T14:15:26,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742191_1367 (size=12001) 2024-11-07T14:15:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T14:15:26,523 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:26,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-07T14:15:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T14:15:26,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:26,528 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:26,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:26,530 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/468b57c3566c436281985cc177295fd9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/468b57c3566c436281985cc177295fd9 2024-11-07T14:15:26,537 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/A of 186148ebfb7e3f0985eb6829e86c465f into 468b57c3566c436281985cc177295fd9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:26,537 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:26,538 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/A, priority=13, startTime=1730988925672; duration=0sec 2024-11-07T14:15:26,538 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:26,538 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:A 2024-11-07T14:15:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T14:15:26,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988986633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988986643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988986643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988986643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988986643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:26,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T14:15:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:26,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:26,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T14:15:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:26,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T14:15:26,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/d08112df6788493fa98e844cc60213bd 2024-11-07T14:15:26,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/778a65ff9f604f8b897979b7694ffeb2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/778a65ff9f604f8b897979b7694ffeb2 2024-11-07T14:15:26,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/778a65ff9f604f8b897979b7694ffeb2, entries=250, sequenceid=78, filesize=47.0 K 2024-11-07T14:15:26,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/02a0ce8ad6f7425380d7657613b37443 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/02a0ce8ad6f7425380d7657613b37443 2024-11-07T14:15:26,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/02a0ce8ad6f7425380d7657613b37443, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T14:15:26,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/d08112df6788493fa98e844cc60213bd as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/d08112df6788493fa98e844cc60213bd 2024-11-07T14:15:26,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/d08112df6788493fa98e844cc60213bd, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T14:15:26,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 186148ebfb7e3f0985eb6829e86c465f in 911ms, sequenceid=78, compaction requested=false 2024-11-07T14:15:26,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:26,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T14:15:26,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:26,988 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T14:15:26,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b805683f26cb4745a76021d62c889304_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988925989/Put/seqid=0 2024-11-07T14:15:26,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742192_1368 (size=12154) 2024-11-07T14:15:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,000 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b805683f26cb4745a76021d62c889304_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b805683f26cb4745a76021d62c889304_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:27,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c6a769ad3b664774b5e871a9557c22b0, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:27,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c6a769ad3b664774b5e871a9557c22b0 is 175, key is test_row_0/A:col10/1730988925989/Put/seqid=0 2024-11-07T14:15:27,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742193_1369 (size=30955) 2024-11-07T14:15:27,005 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c6a769ad3b664774b5e871a9557c22b0 2024-11-07T14:15:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/651a799f99a7485e99cf095fc9817535 is 50, key is test_row_0/B:col10/1730988925989/Put/seqid=0 2024-11-07T14:15:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742194_1370 (size=12001) 2024-11-07T14:15:27,015 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/651a799f99a7485e99cf095fc9817535 2024-11-07T14:15:27,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e0ec9c3c0a344f18977dc6e2127dfb51 is 50, key is test_row_0/C:col10/1730988925989/Put/seqid=0 2024-11-07T14:15:27,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742195_1371 (size=12001) 2024-11-07T14:15:27,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e0ec9c3c0a344f18977dc6e2127dfb51 2024-11-07T14:15:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c6a769ad3b664774b5e871a9557c22b0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c6a769ad3b664774b5e871a9557c22b0 2024-11-07T14:15:27,044 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c6a769ad3b664774b5e871a9557c22b0, entries=150, sequenceid=95, filesize=30.2 K 2024-11-07T14:15:27,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/651a799f99a7485e99cf095fc9817535 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/651a799f99a7485e99cf095fc9817535 2024-11-07T14:15:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,048 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/651a799f99a7485e99cf095fc9817535, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T14:15:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e0ec9c3c0a344f18977dc6e2127dfb51 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e0ec9c3c0a344f18977dc6e2127dfb51 2024-11-07T14:15:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e0ec9c3c0a344f18977dc6e2127dfb51, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T14:15:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 186148ebfb7e3f0985eb6829e86c465f in 65ms, sequenceid=95, compaction requested=true 2024-11-07T14:15:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-07T14:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-07T14:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-07T14:15:27,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 527 msec 2024-11-07T14:15:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 533 msec 2024-11-07T14:15:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T14:15:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,153 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:27,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-07T14:15:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,155 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:27,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T14:15:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T14:15:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:27,282 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:27,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:15:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110768653f413ddd412a8032691a6bfaa55d_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988927281/Put/seqid=0 2024-11-07T14:15:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742196_1372 (size=19474) 2024-11-07T14:15:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T14:15:27,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:27,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988987335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988987336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988987337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988987344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988987345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988987452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988987452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988987452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T14:15:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:27,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988987462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988987462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T14:15:27,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T14:15:27,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:27,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988987662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988987662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988987662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988987672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988987682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:27,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110768653f413ddd412a8032691a6bfaa55d_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110768653f413ddd412a8032691a6bfaa55d_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:27,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/2b8332a7b8e84dc1a653ef87b95a26c7, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:27,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/2b8332a7b8e84dc1a653ef87b95a26c7 is 175, key is test_row_0/A:col10/1730988927281/Put/seqid=0 2024-11-07T14:15:27,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742197_1373 (size=56733) 2024-11-07T14:15:27,706 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/2b8332a7b8e84dc1a653ef87b95a26c7 2024-11-07T14:15:27,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/cf2d86234ac74b36b6dd060acbcff5eb is 50, key is test_row_0/B:col10/1730988927281/Put/seqid=0 2024-11-07T14:15:27,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742198_1374 (size=12001) 2024-11-07T14:15:27,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T14:15:27,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:27,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T14:15:27,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T14:15:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:27,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:27,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988987972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988987982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988987982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988987982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988988002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T14:15:28,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:28,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:28,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:28,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:28,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:28,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/cf2d86234ac74b36b6dd060acbcff5eb 2024-11-07T14:15:28,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e05f64a015a949dbb486ebd978c277a0 is 50, key is test_row_0/C:col10/1730988927281/Put/seqid=0 2024-11-07T14:15:28,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742199_1375 (size=12001) 2024-11-07T14:15:28,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e05f64a015a949dbb486ebd978c277a0 2024-11-07T14:15:28,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/2b8332a7b8e84dc1a653ef87b95a26c7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/2b8332a7b8e84dc1a653ef87b95a26c7 2024-11-07T14:15:28,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/2b8332a7b8e84dc1a653ef87b95a26c7, entries=300, sequenceid=106, filesize=55.4 K 2024-11-07T14:15:28,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/cf2d86234ac74b36b6dd060acbcff5eb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/cf2d86234ac74b36b6dd060acbcff5eb 2024-11-07T14:15:28,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/cf2d86234ac74b36b6dd060acbcff5eb, entries=150, sequenceid=106, filesize=11.7 K 2024-11-07T14:15:28,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e05f64a015a949dbb486ebd978c277a0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e05f64a015a949dbb486ebd978c277a0 2024-11-07T14:15:28,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e05f64a015a949dbb486ebd978c277a0, entries=150, sequenceid=106, filesize=11.7 K 2024-11-07T14:15:28,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 186148ebfb7e3f0985eb6829e86c465f in 867ms, sequenceid=106, compaction requested=true 2024-11-07T14:15:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:28,150 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:28,150 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:28,151 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:28,151 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/B is initiating minor compaction (all files) 2024-11-07T14:15:28,152 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/B in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:28,152 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c9685e8ced7d4789b945eb5991c0b2e1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/02a0ce8ad6f7425380d7657613b37443, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/651a799f99a7485e99cf095fc9817535, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/cf2d86234ac74b36b6dd060acbcff5eb] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=47.0 K 2024-11-07T14:15:28,153 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c9685e8ced7d4789b945eb5991c0b2e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988924622 2024-11-07T14:15:28,153 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166885 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:28,153 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 02a0ce8ad6f7425380d7657613b37443, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730988924785 2024-11-07T14:15:28,153 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/A is initiating minor compaction (all files) 2024-11-07T14:15:28,153 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/A in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:28,153 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/468b57c3566c436281985cc177295fd9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/778a65ff9f604f8b897979b7694ffeb2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c6a769ad3b664774b5e871a9557c22b0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/2b8332a7b8e84dc1a653ef87b95a26c7] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=163.0 K 2024-11-07T14:15:28,153 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 651a799f99a7485e99cf095fc9817535, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730988925978 2024-11-07T14:15:28,153 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:28,154 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/468b57c3566c436281985cc177295fd9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/778a65ff9f604f8b897979b7694ffeb2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c6a769ad3b664774b5e871a9557c22b0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/2b8332a7b8e84dc1a653ef87b95a26c7] 2024-11-07T14:15:28,154 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2d86234ac74b36b6dd060acbcff5eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730988927279 2024-11-07T14:15:28,154 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 468b57c3566c436281985cc177295fd9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988924622 2024-11-07T14:15:28,154 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 778a65ff9f604f8b897979b7694ffeb2, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730988924785 2024-11-07T14:15:28,154 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a769ad3b664774b5e871a9557c22b0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730988925978 2024-11-07T14:15:28,155 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b8332a7b8e84dc1a653ef87b95a26c7, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730988927210 2024-11-07T14:15:28,160 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:28,161 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#B#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:28,162 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c483b2fd065749dbb1bb089c32fdff35 is 50, key is test_row_0/B:col10/1730988927281/Put/seqid=0 2024-11-07T14:15:28,170 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107edb7b44c7166428dbb1c0c684606d8cb_186148ebfb7e3f0985eb6829e86c465f store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:28,172 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107edb7b44c7166428dbb1c0c684606d8cb_186148ebfb7e3f0985eb6829e86c465f, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:28,172 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107edb7b44c7166428dbb1c0c684606d8cb_186148ebfb7e3f0985eb6829e86c465f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:28,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742200_1376 (size=12241) 2024-11-07T14:15:28,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742201_1377 (size=4469) 2024-11-07T14:15:28,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T14:15:28,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:28,225 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:15:28,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b62ee4824ad54098a3e25367308aa233_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988927334/Put/seqid=0 2024-11-07T14:15:28,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742202_1378 (size=12204) 2024-11-07T14:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T14:15:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:28,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:28,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988988504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988988508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988988510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988988511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988988523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,586 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c483b2fd065749dbb1bb089c32fdff35 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c483b2fd065749dbb1bb089c32fdff35 2024-11-07T14:15:28,589 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/B of 186148ebfb7e3f0985eb6829e86c465f into c483b2fd065749dbb1bb089c32fdff35(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:28,589 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:28,589 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/B, priority=12, startTime=1730988928150; duration=0sec 2024-11-07T14:15:28,589 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:28,589 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:B 2024-11-07T14:15:28,590 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:28,590 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:28,590 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/C is initiating minor compaction (all files) 2024-11-07T14:15:28,590 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/C in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:28,590 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/271bdd73d1f14660adb218583abdec67, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/d08112df6788493fa98e844cc60213bd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e0ec9c3c0a344f18977dc6e2127dfb51, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e05f64a015a949dbb486ebd978c277a0] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=47.0 K 2024-11-07T14:15:28,591 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 271bdd73d1f14660adb218583abdec67, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988924622 2024-11-07T14:15:28,591 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d08112df6788493fa98e844cc60213bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730988924785 2024-11-07T14:15:28,591 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e0ec9c3c0a344f18977dc6e2127dfb51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730988925978 2024-11-07T14:15:28,591 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e05f64a015a949dbb486ebd978c277a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730988927279 2024-11-07T14:15:28,592 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#A#compaction#326 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:28,593 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c0af3b5e51fe463a8b65f671ce945ea9 is 175, key is test_row_0/A:col10/1730988927281/Put/seqid=0 2024-11-07T14:15:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742203_1379 (size=31195) 2024-11-07T14:15:28,600 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:28,600 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/5144b805e00e4e639cd421932a8c1756 is 50, key is test_row_0/C:col10/1730988927281/Put/seqid=0 2024-11-07T14:15:28,602 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c0af3b5e51fe463a8b65f671ce945ea9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c0af3b5e51fe463a8b65f671ce945ea9 2024-11-07T14:15:28,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742204_1380 (size=12241) 2024-11-07T14:15:28,608 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/A of 186148ebfb7e3f0985eb6829e86c465f into c0af3b5e51fe463a8b65f671ce945ea9(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:28,608 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:28,608 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/A, priority=12, startTime=1730988928150; duration=0sec 2024-11-07T14:15:28,608 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:28,608 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:A 2024-11-07T14:15:28,611 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/5144b805e00e4e639cd421932a8c1756 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5144b805e00e4e639cd421932a8c1756 2024-11-07T14:15:28,616 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/C of 186148ebfb7e3f0985eb6829e86c465f into 5144b805e00e4e639cd421932a8c1756(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:28,616 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:28,616 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/C, priority=12, startTime=1730988928150; duration=0sec 2024-11-07T14:15:28,616 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:28,616 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:C 2024-11-07T14:15:28,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988988622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988988623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988988632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988988632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,640 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b62ee4824ad54098a3e25367308aa233_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b62ee4824ad54098a3e25367308aa233_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/f14dbd5b9854461496ec25660083aa20, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:28,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/f14dbd5b9854461496ec25660083aa20 is 175, key is test_row_0/A:col10/1730988927334/Put/seqid=0 2024-11-07T14:15:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742205_1381 (size=31005) 2024-11-07T14:15:28,647 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/f14dbd5b9854461496ec25660083aa20 2024-11-07T14:15:28,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6 is 50, key is test_row_0/B:col10/1730988927334/Put/seqid=0 2024-11-07T14:15:28,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742206_1382 (size=12051) 2024-11-07T14:15:28,665 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6 2024-11-07T14:15:28,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/a24fa6cd78bf46a5b46784d8efd5dfb9 is 50, key is test_row_0/C:col10/1730988927334/Put/seqid=0 2024-11-07T14:15:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742207_1383 (size=12051) 2024-11-07T14:15:28,698 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/a24fa6cd78bf46a5b46784d8efd5dfb9 2024-11-07T14:15:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/f14dbd5b9854461496ec25660083aa20 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/f14dbd5b9854461496ec25660083aa20 2024-11-07T14:15:28,710 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/f14dbd5b9854461496ec25660083aa20, entries=150, sequenceid=131, filesize=30.3 K 2024-11-07T14:15:28,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6 2024-11-07T14:15:28,716 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6, entries=150, sequenceid=131, filesize=11.8 K 2024-11-07T14:15:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/a24fa6cd78bf46a5b46784d8efd5dfb9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/a24fa6cd78bf46a5b46784d8efd5dfb9 2024-11-07T14:15:28,728 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/a24fa6cd78bf46a5b46784d8efd5dfb9, entries=150, sequenceid=131, filesize=11.8 K 2024-11-07T14:15:28,729 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 186148ebfb7e3f0985eb6829e86c465f in 504ms, sequenceid=131, compaction requested=false 2024-11-07T14:15:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-07T14:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-07T14:15:28,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-07T14:15:28,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-11-07T14:15:28,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5800 sec 2024-11-07T14:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:28,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:15:28,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f79ccdc7c50245e3bc8ea244f74898f9_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988928497/Put/seqid=0 2024-11-07T14:15:28,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742208_1384 (size=12304) 2024-11-07T14:15:28,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988988892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988988899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988988902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:28,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988988903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988989012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988989022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988989022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988989022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988989222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988989232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988989232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988989232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,269 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:29,272 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f79ccdc7c50245e3bc8ea244f74898f9_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f79ccdc7c50245e3bc8ea244f74898f9_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:29,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c934effe2d3a4360b200509f555b5bc8, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:29,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c934effe2d3a4360b200509f555b5bc8 is 175, key is test_row_0/A:col10/1730988928497/Put/seqid=0 2024-11-07T14:15:29,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742209_1385 (size=31105) 2024-11-07T14:15:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T14:15:29,302 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:29,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-07T14:15:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T14:15:29,304 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:29,305 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:29,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T14:15:29,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T14:15:29,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:29,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:29,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:29,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988989532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988989532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,533 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 186148ebfb7e3f0985eb6829e86c465f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:15:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988989542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988989542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988989542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:29,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T14:15:29,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:29,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:29,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:29,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T14:15:29,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c934effe2d3a4360b200509f555b5bc8 2024-11-07T14:15:29,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/f878669e120b4d47a88764acde3d92e4 is 50, key is test_row_0/B:col10/1730988928497/Put/seqid=0 2024-11-07T14:15:29,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742210_1386 (size=12151) 2024-11-07T14:15:29,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/f878669e120b4d47a88764acde3d92e4 2024-11-07T14:15:29,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/86ad32c2c51a46e58a6a45d12f3b368c is 50, key is test_row_0/C:col10/1730988928497/Put/seqid=0 2024-11-07T14:15:29,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742211_1387 (size=12151) 2024-11-07T14:15:29,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T14:15:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:29,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T14:15:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:29,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:29,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:29,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T14:15:30,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988990042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:30,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988990053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:30,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:30,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:30,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:30,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988990063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:30,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988990063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:30,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T14:15:30,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:30,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:30,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/86ad32c2c51a46e58a6a45d12f3b368c 2024-11-07T14:15:30,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c934effe2d3a4360b200509f555b5bc8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c934effe2d3a4360b200509f555b5bc8 2024-11-07T14:15:30,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c934effe2d3a4360b200509f555b5bc8, entries=150, sequenceid=146, filesize=30.4 K 2024-11-07T14:15:30,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/f878669e120b4d47a88764acde3d92e4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/f878669e120b4d47a88764acde3d92e4 2024-11-07T14:15:30,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/f878669e120b4d47a88764acde3d92e4, entries=150, sequenceid=146, filesize=11.9 K 2024-11-07T14:15:30,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/86ad32c2c51a46e58a6a45d12f3b368c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/86ad32c2c51a46e58a6a45d12f3b368c 2024-11-07T14:15:30,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T14:15:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:30,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/86ad32c2c51a46e58a6a45d12f3b368c, entries=150, sequenceid=146, filesize=11.9 K 2024-11-07T14:15:30,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 186148ebfb7e3f0985eb6829e86c465f in 1400ms, sequenceid=146, compaction requested=true 2024-11-07T14:15:30,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:30,240 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:30,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:30,242 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:30,242 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/A is initiating minor compaction (all files) 2024-11-07T14:15:30,242 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/A in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,243 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c0af3b5e51fe463a8b65f671ce945ea9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/f14dbd5b9854461496ec25660083aa20, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c934effe2d3a4360b200509f555b5bc8] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=91.1 K 2024-11-07T14:15:30,243 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,243 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c0af3b5e51fe463a8b65f671ce945ea9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/f14dbd5b9854461496ec25660083aa20, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c934effe2d3a4360b200509f555b5bc8] 2024-11-07T14:15:30,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:30,244 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0af3b5e51fe463a8b65f671ce945ea9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730988927279 2024-11-07T14:15:30,244 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:30,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:30,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:30,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:30,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:30,245 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting f14dbd5b9854461496ec25660083aa20, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730988927334 2024-11-07T14:15:30,245 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c934effe2d3a4360b200509f555b5bc8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1730988928497 2024-11-07T14:15:30,246 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:30,246 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/B is initiating minor compaction (all files) 2024-11-07T14:15:30,246 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/B in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,246 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c483b2fd065749dbb1bb089c32fdff35, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/f878669e120b4d47a88764acde3d92e4] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=35.6 K 2024-11-07T14:15:30,246 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c483b2fd065749dbb1bb089c32fdff35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730988927279 2024-11-07T14:15:30,247 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting a7bb590f8a3b46b2b8a7a2bf4ec0d8c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730988927334 2024-11-07T14:15:30,247 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f878669e120b4d47a88764acde3d92e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1730988928497 2024-11-07T14:15:30,260 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:30,261 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:30,261 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/72dc388392ec40d8959c9b659045ce9b is 50, key is test_row_0/B:col10/1730988928497/Put/seqid=0 2024-11-07T14:15:30,267 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110730f14a829943414092cd542dad7a61bf_186148ebfb7e3f0985eb6829e86c465f store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:30,269 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110730f14a829943414092cd542dad7a61bf_186148ebfb7e3f0985eb6829e86c465f, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:30,269 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110730f14a829943414092cd542dad7a61bf_186148ebfb7e3f0985eb6829e86c465f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:30,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742212_1388 (size=12493) 2024-11-07T14:15:30,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742213_1389 (size=4469) 2024-11-07T14:15:30,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T14:15:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,384 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:15:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c86c6b9dad254f8b9396deb4d8884f0e_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988928901/Put/seqid=0 2024-11-07T14:15:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742214_1390 (size=12304) 2024-11-07T14:15:30,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:30,401 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c86c6b9dad254f8b9396deb4d8884f0e_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c86c6b9dad254f8b9396deb4d8884f0e_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/33334af101584e63a1b863f533e0203d, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/33334af101584e63a1b863f533e0203d is 175, key is test_row_0/A:col10/1730988928901/Put/seqid=0 2024-11-07T14:15:30,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742215_1391 (size=31105) 2024-11-07T14:15:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T14:15:30,689 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/72dc388392ec40d8959c9b659045ce9b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72dc388392ec40d8959c9b659045ce9b 2024-11-07T14:15:30,694 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/B of 186148ebfb7e3f0985eb6829e86c465f into 72dc388392ec40d8959c9b659045ce9b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:30,694 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:30,694 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/B, priority=13, startTime=1730988930244; duration=0sec 2024-11-07T14:15:30,694 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:30,694 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:B 2024-11-07T14:15:30,694 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:30,697 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:30,697 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/C is initiating minor compaction (all files) 2024-11-07T14:15:30,697 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#A#compaction#335 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:30,697 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/C in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:30,697 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5144b805e00e4e639cd421932a8c1756, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/a24fa6cd78bf46a5b46784d8efd5dfb9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/86ad32c2c51a46e58a6a45d12f3b368c] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=35.6 K 2024-11-07T14:15:30,697 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/5950715a3bab4eb180814e2949fb35a4 is 175, key is test_row_0/A:col10/1730988928497/Put/seqid=0 2024-11-07T14:15:30,698 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 5144b805e00e4e639cd421932a8c1756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1730988927279 2024-11-07T14:15:30,699 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting a24fa6cd78bf46a5b46784d8efd5dfb9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730988927334 2024-11-07T14:15:30,699 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ad32c2c51a46e58a6a45d12f3b368c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1730988928497 2024-11-07T14:15:30,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742216_1392 (size=31447) 2024-11-07T14:15:30,707 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#C#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:30,707 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/2788ebc33cf7480fb140cd104d2a7b9f is 50, key is test_row_0/C:col10/1730988928497/Put/seqid=0 2024-11-07T14:15:30,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742217_1393 (size=12493) 2024-11-07T14:15:30,719 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/2788ebc33cf7480fb140cd104d2a7b9f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2788ebc33cf7480fb140cd104d2a7b9f 2024-11-07T14:15:30,724 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/C of 186148ebfb7e3f0985eb6829e86c465f into 2788ebc33cf7480fb140cd104d2a7b9f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:30,724 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:30,724 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/C, priority=13, startTime=1730988930244; duration=0sec 2024-11-07T14:15:30,724 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:30,724 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:C 2024-11-07T14:15:30,808 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/33334af101584e63a1b863f533e0203d 2024-11-07T14:15:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/33538fe7e6b345f294d7756671759b49 is 50, key is test_row_0/B:col10/1730988928901/Put/seqid=0 2024-11-07T14:15:30,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742218_1394 (size=12151) 2024-11-07T14:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:31,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:31,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988991084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988991086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988991087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988991088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,105 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/5950715a3bab4eb180814e2949fb35a4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5950715a3bab4eb180814e2949fb35a4 2024-11-07T14:15:31,108 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/A of 186148ebfb7e3f0985eb6829e86c465f into 5950715a3bab4eb180814e2949fb35a4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:31,108 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:31,108 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/A, priority=13, startTime=1730988930240; duration=0sec 2024-11-07T14:15:31,108 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:31,108 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:A 2024-11-07T14:15:31,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988991202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988991202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988991202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988991202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,223 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/33538fe7e6b345f294d7756671759b49 2024-11-07T14:15:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e7daff4cdcc34ceb8cfb5e4f39257a91 is 50, key is test_row_0/C:col10/1730988928901/Put/seqid=0 2024-11-07T14:15:31,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742219_1395 (size=12151) 2024-11-07T14:15:31,236 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e7daff4cdcc34ceb8cfb5e4f39257a91 2024-11-07T14:15:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/33334af101584e63a1b863f533e0203d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/33334af101584e63a1b863f533e0203d 2024-11-07T14:15:31,243 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/33334af101584e63a1b863f533e0203d, entries=150, sequenceid=170, filesize=30.4 K 2024-11-07T14:15:31,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/33538fe7e6b345f294d7756671759b49 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33538fe7e6b345f294d7756671759b49 2024-11-07T14:15:31,246 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33538fe7e6b345f294d7756671759b49, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T14:15:31,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/e7daff4cdcc34ceb8cfb5e4f39257a91 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e7daff4cdcc34ceb8cfb5e4f39257a91 2024-11-07T14:15:31,252 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e7daff4cdcc34ceb8cfb5e4f39257a91, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T14:15:31,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 186148ebfb7e3f0985eb6829e86c465f in 868ms, sequenceid=170, compaction requested=false 2024-11-07T14:15:31,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:31,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:31,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-07T14:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-07T14:15:31,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-07T14:15:31,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9490 sec 2024-11-07T14:15:31,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.9530 sec 2024-11-07T14:15:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:31,415 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:31,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:15:31,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aaaa5d8b53bd46228f48c3f8ca9099c8_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988931415/Put/seqid=0 2024-11-07T14:15:31,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742220_1396 (size=17284) 2024-11-07T14:15:31,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:31,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aaaa5d8b53bd46228f48c3f8ca9099c8_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aaaa5d8b53bd46228f48c3f8ca9099c8_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:31,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/7b08d3f31a364beca0bf308eeca4613d, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:31,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/7b08d3f31a364beca0bf308eeca4613d is 175, key is test_row_0/A:col10/1730988931415/Put/seqid=0 2024-11-07T14:15:31,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742221_1397 (size=48389) 2024-11-07T14:15:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T14:15:31,452 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:31,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-07T14:15:31,454 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T14:15:31,455 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:31,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988991447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988991448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988991449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988991455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988991552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,553 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 186148ebfb7e3f0985eb6829e86c465f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:15:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T14:15:31,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988991562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988991562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988991562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988991562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:31,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:31,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:31,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:31,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:31,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:31,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:31,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:31,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T14:15:31,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988991772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988991782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988991782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988991782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:31,835 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/7b08d3f31a364beca0bf308eeca4613d 2024-11-07T14:15:31,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/b51f34d171224c92a4b9594e7b49f17d is 50, key is test_row_0/B:col10/1730988931415/Put/seqid=0 2024-11-07T14:15:31,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742222_1398 (size=12151) 2024-11-07T14:15:31,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:31,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:31,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:31,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:31,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:31,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T14:15:32,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988992082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:32,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988992092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:32,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988992092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:32,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988992092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:32,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:32,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/b51f34d171224c92a4b9594e7b49f17d 2024-11-07T14:15:32,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/447a586390c2476680132185ee807f00 is 50, key is test_row_0/C:col10/1730988931415/Put/seqid=0 2024-11-07T14:15:32,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742223_1399 (size=12151) 2024-11-07T14:15:32,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:32,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:32,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T14:15:32,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988992592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:32,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988992602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:32,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988992602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:32,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988992602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:32,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/447a586390c2476680132185ee807f00 2024-11-07T14:15:32,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/7b08d3f31a364beca0bf308eeca4613d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/7b08d3f31a364beca0bf308eeca4613d 2024-11-07T14:15:32,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/7b08d3f31a364beca0bf308eeca4613d, entries=250, sequenceid=187, filesize=47.3 K 2024-11-07T14:15:32,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/b51f34d171224c92a4b9594e7b49f17d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/b51f34d171224c92a4b9594e7b49f17d 2024-11-07T14:15:32,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:32,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:32,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/b51f34d171224c92a4b9594e7b49f17d, entries=150, sequenceid=187, filesize=11.9 K 2024-11-07T14:15:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/447a586390c2476680132185ee807f00 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/447a586390c2476680132185ee807f00 2024-11-07T14:15:32,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/447a586390c2476680132185ee807f00, entries=150, sequenceid=187, filesize=11.9 K 2024-11-07T14:15:32,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 186148ebfb7e3f0985eb6829e86c465f in 1274ms, sequenceid=187, compaction requested=true 2024-11-07T14:15:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:15:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T14:15:32,690 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:32,690 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:32,691 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:32,691 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/A is initiating minor compaction (all files) 2024-11-07T14:15:32,691 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/A in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,691 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5950715a3bab4eb180814e2949fb35a4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/33334af101584e63a1b863f533e0203d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/7b08d3f31a364beca0bf308eeca4613d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=108.3 K 2024-11-07T14:15:32,691 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,691 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5950715a3bab4eb180814e2949fb35a4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/33334af101584e63a1b863f533e0203d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/7b08d3f31a364beca0bf308eeca4613d] 2024-11-07T14:15:32,691 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:32,691 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/C is initiating minor compaction (all files) 2024-11-07T14:15:32,691 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/C in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,691 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2788ebc33cf7480fb140cd104d2a7b9f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e7daff4cdcc34ceb8cfb5e4f39257a91, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/447a586390c2476680132185ee807f00] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=35.9 K 2024-11-07T14:15:32,692 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5950715a3bab4eb180814e2949fb35a4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1730988928497 2024-11-07T14:15:32,692 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2788ebc33cf7480fb140cd104d2a7b9f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1730988928497 2024-11-07T14:15:32,692 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33334af101584e63a1b863f533e0203d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730988928897 2024-11-07T14:15:32,692 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b08d3f31a364beca0bf308eeca4613d, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1730988931081 2024-11-07T14:15:32,692 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e7daff4cdcc34ceb8cfb5e4f39257a91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730988928897 2024-11-07T14:15:32,693 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 447a586390c2476680132185ee807f00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1730988931081 2024-11-07T14:15:32,702 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#C#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:32,703 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/5cb327d8524e42d0bf5cf010b8518e18 is 50, key is test_row_0/C:col10/1730988931415/Put/seqid=0 2024-11-07T14:15:32,711 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:32,722 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107cdfbba15eb7d43f7a52ad16817d093cb_186148ebfb7e3f0985eb6829e86c465f store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:32,724 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107cdfbba15eb7d43f7a52ad16817d093cb_186148ebfb7e3f0985eb6829e86c465f, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:32,725 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cdfbba15eb7d43f7a52ad16817d093cb_186148ebfb7e3f0985eb6829e86c465f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:32,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742224_1400 (size=12595) 2024-11-07T14:15:32,736 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/5cb327d8524e42d0bf5cf010b8518e18 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5cb327d8524e42d0bf5cf010b8518e18 2024-11-07T14:15:32,741 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/C of 186148ebfb7e3f0985eb6829e86c465f into 5cb327d8524e42d0bf5cf010b8518e18(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:32,741 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:32,741 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/C, priority=13, startTime=1730988932689; duration=0sec 2024-11-07T14:15:32,741 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:32,741 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:C 2024-11-07T14:15:32,741 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:32,742 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:32,742 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/B is initiating minor compaction (all files) 2024-11-07T14:15:32,742 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/B in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,742 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72dc388392ec40d8959c9b659045ce9b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33538fe7e6b345f294d7756671759b49, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/b51f34d171224c92a4b9594e7b49f17d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=35.9 K 2024-11-07T14:15:32,742 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 72dc388392ec40d8959c9b659045ce9b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1730988928497 2024-11-07T14:15:32,742 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 33538fe7e6b345f294d7756671759b49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730988928897 2024-11-07T14:15:32,742 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b51f34d171224c92a4b9594e7b49f17d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1730988931081 2024-11-07T14:15:32,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742225_1401 (size=4469) 2024-11-07T14:15:32,748 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#A#compaction#344 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:32,748 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/4523f16379ba4cb6aa984ed805a5a510 is 175, key is test_row_0/A:col10/1730988931415/Put/seqid=0 2024-11-07T14:15:32,752 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:32,752 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/3bbc3a8899e14ac1a851b801241e0138 is 50, key is test_row_0/B:col10/1730988931415/Put/seqid=0 2024-11-07T14:15:32,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742226_1402 (size=31549) 2024-11-07T14:15:32,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742227_1403 (size=12595) 2024-11-07T14:15:32,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T14:15:32,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:32,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T14:15:32,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076ce337a753ed4c8db1c8f5c6a794c6c6_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988931438/Put/seqid=0 2024-11-07T14:15:32,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742228_1404 (size=12304) 2024-11-07T14:15:33,188 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/4523f16379ba4cb6aa984ed805a5a510 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/4523f16379ba4cb6aa984ed805a5a510 2024-11-07T14:15:33,192 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/3bbc3a8899e14ac1a851b801241e0138 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/3bbc3a8899e14ac1a851b801241e0138 2024-11-07T14:15:33,192 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/A of 186148ebfb7e3f0985eb6829e86c465f into 4523f16379ba4cb6aa984ed805a5a510(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:33,192 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:33,192 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/A, priority=13, startTime=1730988932689; duration=0sec 2024-11-07T14:15:33,192 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:33,192 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:A 2024-11-07T14:15:33,196 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/B of 186148ebfb7e3f0985eb6829e86c465f into 3bbc3a8899e14ac1a851b801241e0138(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:33,196 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:33,196 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/B, priority=13, startTime=1730988932689; duration=0sec 2024-11-07T14:15:33,196 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:33,196 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:B 2024-11-07T14:15:33,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:33,252 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076ce337a753ed4c8db1c8f5c6a794c6c6_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ce337a753ed4c8db1c8f5c6a794c6c6_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:33,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/e54b959314dc4c4aaad2fbaec5dcf682, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:33,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/e54b959314dc4c4aaad2fbaec5dcf682 is 175, key is test_row_0/A:col10/1730988931438/Put/seqid=0 2024-11-07T14:15:33,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742229_1405 (size=31105) 2024-11-07T14:15:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T14:15:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:33,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:33,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988993635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988993643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988993643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988993644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,659 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/e54b959314dc4c4aaad2fbaec5dcf682 2024-11-07T14:15:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/1437b30e50ea456d8e4d013eb8199fa2 is 50, key is test_row_0/B:col10/1730988931438/Put/seqid=0 2024-11-07T14:15:33,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742230_1406 (size=12151) 2024-11-07T14:15:33,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988993752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988993752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988993752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988993762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988993962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988993962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988993973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988993973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,067 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/1437b30e50ea456d8e4d013eb8199fa2 2024-11-07T14:15:34,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/c844064d0cc64dae9aac549f3f958dd9 is 50, key is test_row_0/C:col10/1730988931438/Put/seqid=0 2024-11-07T14:15:34,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742231_1407 (size=12151) 2024-11-07T14:15:34,090 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/c844064d0cc64dae9aac549f3f958dd9 2024-11-07T14:15:34,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/e54b959314dc4c4aaad2fbaec5dcf682 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/e54b959314dc4c4aaad2fbaec5dcf682 2024-11-07T14:15:34,098 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/e54b959314dc4c4aaad2fbaec5dcf682, entries=150, sequenceid=210, filesize=30.4 K 2024-11-07T14:15:34,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/1437b30e50ea456d8e4d013eb8199fa2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/1437b30e50ea456d8e4d013eb8199fa2 2024-11-07T14:15:34,103 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/1437b30e50ea456d8e4d013eb8199fa2, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T14:15:34,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/c844064d0cc64dae9aac549f3f958dd9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/c844064d0cc64dae9aac549f3f958dd9 2024-11-07T14:15:34,106 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/c844064d0cc64dae9aac549f3f958dd9, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T14:15:34,107 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 186148ebfb7e3f0985eb6829e86c465f in 1276ms, sequenceid=210, compaction requested=false 2024-11-07T14:15:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-07T14:15:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-07T14:15:34,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-07T14:15:34,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6540 sec 2024-11-07T14:15:34,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.6590 sec 2024-11-07T14:15:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:34,275 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:34,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T14:15:34,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076b85d187ca1e429ea54b0d96bade591b_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988933630/Put/seqid=0 2024-11-07T14:15:34,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742232_1408 (size=17284) 2024-11-07T14:15:34,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988994323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988994324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988994327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988994329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988994442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988994443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988994443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988994443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988994662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988994663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988994663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988994663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:34,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076b85d187ca1e429ea54b0d96bade591b_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076b85d187ca1e429ea54b0d96bade591b_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:34,701 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/60737df2017c4f2db4aa3c1d981e9aee, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:34,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/60737df2017c4f2db4aa3c1d981e9aee is 175, key is test_row_0/A:col10/1730988933630/Put/seqid=0 2024-11-07T14:15:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742233_1409 (size=48389) 2024-11-07T14:15:34,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988994972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988994972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988994981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988994982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:35,125 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/60737df2017c4f2db4aa3c1d981e9aee 2024-11-07T14:15:35,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/4e058a900103421db085d2a29c763c7c is 50, key is test_row_0/B:col10/1730988933630/Put/seqid=0 2024-11-07T14:15:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742234_1410 (size=12151) 2024-11-07T14:15:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988995492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:35,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988995493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:35,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988995494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:35,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988995503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:35,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/4e058a900103421db085d2a29c763c7c 2024-11-07T14:15:35,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/35fb7e594f94450db34b4596338482b0 is 50, key is test_row_0/C:col10/1730988933630/Put/seqid=0 2024-11-07T14:15:35,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52402 deadline: 1730988995594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:35,612 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 186148ebfb7e3f0985eb6829e86c465f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:15:35,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:35,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T14:15:35,623 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:35,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-07T14:15:35,626 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:35,627 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:35,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T14:15:35,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742235_1411 (size=12151) 2024-11-07T14:15:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T14:15:35,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T14:15:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:35,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T14:15:35,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T14:15:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:35,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:36,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/35fb7e594f94450db34b4596338482b0 2024-11-07T14:15:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/60737df2017c4f2db4aa3c1d981e9aee as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/60737df2017c4f2db4aa3c1d981e9aee 2024-11-07T14:15:36,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/60737df2017c4f2db4aa3c1d981e9aee, entries=250, sequenceid=229, filesize=47.3 K 2024-11-07T14:15:36,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/4e058a900103421db085d2a29c763c7c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/4e058a900103421db085d2a29c763c7c 2024-11-07T14:15:36,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/4e058a900103421db085d2a29c763c7c, entries=150, sequenceid=229, filesize=11.9 K 2024-11-07T14:15:36,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/35fb7e594f94450db34b4596338482b0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/35fb7e594f94450db34b4596338482b0 2024-11-07T14:15:36,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/35fb7e594f94450db34b4596338482b0, entries=150, sequenceid=229, filesize=11.9 K 2024-11-07T14:15:36,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 186148ebfb7e3f0985eb6829e86c465f in 1823ms, sequenceid=229, compaction requested=true 2024-11-07T14:15:36,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:36,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T14:15:36,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:36,099 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:36,099 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T14:15:36,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:36,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:36,099 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:36,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:36,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:36,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:36,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:36,101 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:36,101 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/A is initiating minor compaction (all files) 2024-11-07T14:15:36,101 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/A in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:36,101 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/4523f16379ba4cb6aa984ed805a5a510, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/e54b959314dc4c4aaad2fbaec5dcf682, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/60737df2017c4f2db4aa3c1d981e9aee] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=108.4 K 2024-11-07T14:15:36,101 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:36,101 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/4523f16379ba4cb6aa984ed805a5a510, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/e54b959314dc4c4aaad2fbaec5dcf682, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/60737df2017c4f2db4aa3c1d981e9aee] 2024-11-07T14:15:36,102 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:36,103 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/B is initiating minor compaction (all files) 2024-11-07T14:15:36,103 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4523f16379ba4cb6aa984ed805a5a510, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1730988931081 2024-11-07T14:15:36,103 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/B in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:36,103 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/3bbc3a8899e14ac1a851b801241e0138, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/1437b30e50ea456d8e4d013eb8199fa2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/4e058a900103421db085d2a29c763c7c] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=36.0 K 2024-11-07T14:15:36,103 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e54b959314dc4c4aaad2fbaec5dcf682, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730988931438 2024-11-07T14:15:36,104 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bbc3a8899e14ac1a851b801241e0138, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1730988931081 2024-11-07T14:15:36,104 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60737df2017c4f2db4aa3c1d981e9aee, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1730988933630 2024-11-07T14:15:36,104 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 1437b30e50ea456d8e4d013eb8199fa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730988931438 2024-11-07T14:15:36,104 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e058a900103421db085d2a29c763c7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1730988933630 2024-11-07T14:15:36,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078a9fa7b556e04a1c8dda02fa32b7af1b_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988934311/Put/seqid=0 2024-11-07T14:15:36,119 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:36,120 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/58df401642c8471888e14606d20ccfac is 50, key is test_row_0/B:col10/1730988933630/Put/seqid=0 2024-11-07T14:15:36,120 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:36,151 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107bb2adee908a044769067df9f9c2639a9_186148ebfb7e3f0985eb6829e86c465f store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:36,154 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107bb2adee908a044769067df9f9c2639a9_186148ebfb7e3f0985eb6829e86c465f, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:36,154 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bb2adee908a044769067df9f9c2639a9_186148ebfb7e3f0985eb6829e86c465f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:36,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742236_1412 (size=12304) 2024-11-07T14:15:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:36,194 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078a9fa7b556e04a1c8dda02fa32b7af1b_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a9fa7b556e04a1c8dda02fa32b7af1b_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:36,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/df14fe58cc7a436db463459f97f7c8cc, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:36,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/df14fe58cc7a436db463459f97f7c8cc is 175, key is test_row_0/A:col10/1730988934311/Put/seqid=0 2024-11-07T14:15:36,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742237_1413 (size=12697) 2024-11-07T14:15:36,213 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/58df401642c8471888e14606d20ccfac as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/58df401642c8471888e14606d20ccfac 2024-11-07T14:15:36,227 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/B of 186148ebfb7e3f0985eb6829e86c465f into 58df401642c8471888e14606d20ccfac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:36,227 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:36,227 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/B, priority=13, startTime=1730988936099; duration=0sec 2024-11-07T14:15:36,227 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:36,227 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:B 2024-11-07T14:15:36,228 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:36,239 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:36,239 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/C is initiating minor compaction (all files) 2024-11-07T14:15:36,239 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/C in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:36,239 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5cb327d8524e42d0bf5cf010b8518e18, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/c844064d0cc64dae9aac549f3f958dd9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/35fb7e594f94450db34b4596338482b0] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=36.0 K 2024-11-07T14:15:36,240 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb327d8524e42d0bf5cf010b8518e18, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1730988931081 2024-11-07T14:15:36,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742239_1415 (size=31105) 2024-11-07T14:15:36,240 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c844064d0cc64dae9aac549f3f958dd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730988931438 2024-11-07T14:15:36,240 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/df14fe58cc7a436db463459f97f7c8cc 2024-11-07T14:15:36,241 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 35fb7e594f94450db34b4596338482b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1730988933630 2024-11-07T14:15:36,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742238_1414 (size=4469) 2024-11-07T14:15:36,245 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#A#compaction#354 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:36,246 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/773a3a9d8b9f4922912dea7f1849d18e is 175, key is test_row_0/A:col10/1730988933630/Put/seqid=0 2024-11-07T14:15:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T14:15:36,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742240_1416 (size=31651) 2024-11-07T14:15:36,288 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:36,288 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/2779a6246efe4d8ca29c24fe7dfc5bc7 is 50, key is test_row_0/C:col10/1730988933630/Put/seqid=0 2024-11-07T14:15:36,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/33d3af56323a4e8e8ee4b02531291ac9 is 50, key is test_row_0/B:col10/1730988934311/Put/seqid=0 2024-11-07T14:15:36,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742241_1417 (size=12697) 2024-11-07T14:15:36,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742242_1418 (size=12151) 2024-11-07T14:15:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:36,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988996562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988996576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988996579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988996581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988996693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988996693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988996695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,705 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/33d3af56323a4e8e8ee4b02531291ac9 2024-11-07T14:15:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,708 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/2779a6246efe4d8ca29c24fe7dfc5bc7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2779a6246efe4d8ca29c24fe7dfc5bc7 2024-11-07T14:15:36,712 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/773a3a9d8b9f4922912dea7f1849d18e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/773a3a9d8b9f4922912dea7f1849d18e 2024-11-07T14:15:36,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988996713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,739 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/C of 186148ebfb7e3f0985eb6829e86c465f into 2779a6246efe4d8ca29c24fe7dfc5bc7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:36,739 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:36,739 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/C, priority=13, startTime=1730988936099; duration=0sec 2024-11-07T14:15:36,739 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:36,739 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:C 2024-11-07T14:15:36,739 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/A of 186148ebfb7e3f0985eb6829e86c465f into 773a3a9d8b9f4922912dea7f1849d18e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:36,740 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:36,740 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/A, priority=13, startTime=1730988936099; duration=0sec 2024-11-07T14:15:36,740 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:36,740 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:A 2024-11-07T14:15:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T14:15:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/827c759e131c407f90c37b7a2300abe2 is 50, key is test_row_0/C:col10/1730988934311/Put/seqid=0 2024-11-07T14:15:36,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742243_1419 (size=12151) 2024-11-07T14:15:36,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/827c759e131c407f90c37b7a2300abe2 2024-11-07T14:15:36,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/df14fe58cc7a436db463459f97f7c8cc as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/df14fe58cc7a436db463459f97f7c8cc 2024-11-07T14:15:36,793 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/df14fe58cc7a436db463459f97f7c8cc, entries=150, sequenceid=248, filesize=30.4 K 2024-11-07T14:15:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/33d3af56323a4e8e8ee4b02531291ac9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33d3af56323a4e8e8ee4b02531291ac9 2024-11-07T14:15:36,798 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33d3af56323a4e8e8ee4b02531291ac9, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T14:15:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/827c759e131c407f90c37b7a2300abe2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/827c759e131c407f90c37b7a2300abe2 2024-11-07T14:15:36,802 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/827c759e131c407f90c37b7a2300abe2, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T14:15:36,803 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 186148ebfb7e3f0985eb6829e86c465f in 704ms, sequenceid=248, compaction requested=false 2024-11-07T14:15:36,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:36,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:36,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-07T14:15:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-07T14:15:36,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-07T14:15:36,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1780 sec 2024-11-07T14:15:36,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.1830 sec 2024-11-07T14:15:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:36,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:15:36,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107557296566eb945ab972bdaecccb7e626_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988936534/Put/seqid=0 2024-11-07T14:15:36,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742244_1420 (size=17534) 2024-11-07T14:15:36,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:36,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107557296566eb945ab972bdaecccb7e626_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107557296566eb945ab972bdaecccb7e626_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:36,942 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/5ae3d594698e4af0bc37a2e68e3063b8, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:36,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/5ae3d594698e4af0bc37a2e68e3063b8 is 175, key is test_row_0/A:col10/1730988936534/Put/seqid=0 2024-11-07T14:15:36,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742245_1421 (size=48639) 2024-11-07T14:15:36,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988996950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988996962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988996962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988996963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988997072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988997083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988997083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988997083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988997282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988997292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988997303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988997303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/5ae3d594698e4af0bc37a2e68e3063b8 2024-11-07T14:15:37,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/46980c29809d4a8f97208aa5e727a3e8 is 50, key is test_row_0/B:col10/1730988936534/Put/seqid=0 2024-11-07T14:15:37,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742246_1422 (size=12301) 2024-11-07T14:15:37,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/46980c29809d4a8f97208aa5e727a3e8 2024-11-07T14:15:37,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/0efbdd7ba40d41fd801a4cc07a8cb0cf is 50, key is test_row_0/C:col10/1730988936534/Put/seqid=0 2024-11-07T14:15:37,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742247_1423 (size=12301) 2024-11-07T14:15:37,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/0efbdd7ba40d41fd801a4cc07a8cb0cf 2024-11-07T14:15:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/5ae3d594698e4af0bc37a2e68e3063b8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5ae3d594698e4af0bc37a2e68e3063b8 2024-11-07T14:15:37,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5ae3d594698e4af0bc37a2e68e3063b8, entries=250, sequenceid=270, filesize=47.5 K 2024-11-07T14:15:37,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/46980c29809d4a8f97208aa5e727a3e8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/46980c29809d4a8f97208aa5e727a3e8 2024-11-07T14:15:37,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/46980c29809d4a8f97208aa5e727a3e8, entries=150, sequenceid=270, filesize=12.0 K 2024-11-07T14:15:37,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/0efbdd7ba40d41fd801a4cc07a8cb0cf as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/0efbdd7ba40d41fd801a4cc07a8cb0cf 2024-11-07T14:15:37,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/0efbdd7ba40d41fd801a4cc07a8cb0cf, entries=150, sequenceid=270, filesize=12.0 K 2024-11-07T14:15:37,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 186148ebfb7e3f0985eb6829e86c465f in 513ms, sequenceid=270, compaction requested=true 2024-11-07T14:15:37,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:37,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:37,430 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:37,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:37,430 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:37,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:37,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:37,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:37,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:37,431 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:37,431 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/B is initiating minor compaction (all files) 2024-11-07T14:15:37,431 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/B in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:37,431 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/58df401642c8471888e14606d20ccfac, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33d3af56323a4e8e8ee4b02531291ac9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/46980c29809d4a8f97208aa5e727a3e8] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=36.3 K 2024-11-07T14:15:37,431 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111395 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:37,431 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/A is initiating minor compaction (all files) 2024-11-07T14:15:37,431 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/A in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:37,432 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/773a3a9d8b9f4922912dea7f1849d18e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/df14fe58cc7a436db463459f97f7c8cc, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5ae3d594698e4af0bc37a2e68e3063b8] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=108.8 K 2024-11-07T14:15:37,432 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:37,432 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/773a3a9d8b9f4922912dea7f1849d18e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/df14fe58cc7a436db463459f97f7c8cc, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5ae3d594698e4af0bc37a2e68e3063b8] 2024-11-07T14:15:37,432 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 58df401642c8471888e14606d20ccfac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1730988933630 2024-11-07T14:15:37,432 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 773a3a9d8b9f4922912dea7f1849d18e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1730988933630 2024-11-07T14:15:37,433 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d3af56323a4e8e8ee4b02531291ac9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988934311 2024-11-07T14:15:37,433 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting df14fe58cc7a436db463459f97f7c8cc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988934311 2024-11-07T14:15:37,433 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 46980c29809d4a8f97208aa5e727a3e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988936534 2024-11-07T14:15:37,433 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae3d594698e4af0bc37a2e68e3063b8, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988936534 2024-11-07T14:15:37,444 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:37,453 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#B#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:37,454 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/5c3d38b76cdc4da2998a997442d7adb0 is 50, key is test_row_0/B:col10/1730988936534/Put/seqid=0 2024-11-07T14:15:37,459 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411073f04e0336c164be1aa705b075f40b3fb_186148ebfb7e3f0985eb6829e86c465f store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:37,461 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411073f04e0336c164be1aa705b075f40b3fb_186148ebfb7e3f0985eb6829e86c465f, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:37,461 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073f04e0336c164be1aa705b075f40b3fb_186148ebfb7e3f0985eb6829e86c465f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:37,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742248_1424 (size=4469) 2024-11-07T14:15:37,495 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#A#compaction#361 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:37,496 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/19d1e9c1e65f4ecea021791319011455 is 175, key is test_row_0/A:col10/1730988936534/Put/seqid=0 2024-11-07T14:15:37,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742249_1425 (size=12949) 2024-11-07T14:15:37,515 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/5c3d38b76cdc4da2998a997442d7adb0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/5c3d38b76cdc4da2998a997442d7adb0 2024-11-07T14:15:37,524 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/B of 186148ebfb7e3f0985eb6829e86c465f into 5c3d38b76cdc4da2998a997442d7adb0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:37,524 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:37,524 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/B, priority=13, startTime=1730988937430; duration=0sec 2024-11-07T14:15:37,524 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:37,524 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:B 2024-11-07T14:15:37,524 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:37,525 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:37,525 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/C is initiating minor compaction (all files) 2024-11-07T14:15:37,525 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/C in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:37,526 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2779a6246efe4d8ca29c24fe7dfc5bc7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/827c759e131c407f90c37b7a2300abe2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/0efbdd7ba40d41fd801a4cc07a8cb0cf] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=36.3 K 2024-11-07T14:15:37,526 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2779a6246efe4d8ca29c24fe7dfc5bc7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1730988933630 2024-11-07T14:15:37,526 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 827c759e131c407f90c37b7a2300abe2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988934311 2024-11-07T14:15:37,526 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0efbdd7ba40d41fd801a4cc07a8cb0cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988936534 2024-11-07T14:15:37,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742250_1426 (size=31903) 2024-11-07T14:15:37,534 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#C#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:37,535 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/03ccc84f5e734a7d982641d5a9631b77 is 50, key is test_row_0/C:col10/1730988936534/Put/seqid=0 2024-11-07T14:15:37,537 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/19d1e9c1e65f4ecea021791319011455 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/19d1e9c1e65f4ecea021791319011455 2024-11-07T14:15:37,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742251_1427 (size=12949) 2024-11-07T14:15:37,542 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/A of 186148ebfb7e3f0985eb6829e86c465f into 19d1e9c1e65f4ecea021791319011455(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:37,542 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:37,542 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/A, priority=13, startTime=1730988937429; duration=0sec 2024-11-07T14:15:37,542 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:37,542 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:A 2024-11-07T14:15:37,545 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/03ccc84f5e734a7d982641d5a9631b77 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/03ccc84f5e734a7d982641d5a9631b77 2024-11-07T14:15:37,550 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/C of 186148ebfb7e3f0985eb6829e86c465f into 03ccc84f5e734a7d982641d5a9631b77(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:37,550 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:37,550 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/C, priority=13, startTime=1730988937430; duration=0sec 2024-11-07T14:15:37,550 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:37,550 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:C 2024-11-07T14:15:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:37,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:37,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:15:37,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072eb47917e05645a5bb2d5421c3c3f98f_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988936960/Put/seqid=0 2024-11-07T14:15:37,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742252_1428 (size=14994) 2024-11-07T14:15:37,616 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:37,619 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072eb47917e05645a5bb2d5421c3c3f98f_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072eb47917e05645a5bb2d5421c3c3f98f_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:37,619 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/d17f5145b0c64b5a99caad241cceaa16, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:37,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/d17f5145b0c64b5a99caad241cceaa16 is 175, key is test_row_0/A:col10/1730988936960/Put/seqid=0 2024-11-07T14:15:37,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742253_1429 (size=39949) 2024-11-07T14:15:37,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988997664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988997666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988997667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988997667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T14:15:37,773 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:37,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-07T14:15:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T14:15:37,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:37,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:37,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:37,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988997773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988997782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988997782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988997782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T14:15:37,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T14:15:37,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:37,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:37,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:37,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:37,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988997992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988997992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988997993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988997993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,024 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/d17f5145b0c64b5a99caad241cceaa16 2024-11-07T14:15:38,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c205332c8d9a4dd0bf0a5ed581785f4b is 50, key is test_row_0/B:col10/1730988936960/Put/seqid=0 2024-11-07T14:15:38,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742254_1430 (size=12301) 2024-11-07T14:15:38,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T14:15:38,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:38,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:38,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:38,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T14:15:38,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T14:15:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:38,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988998303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988998303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988998303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988998312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T14:15:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:38,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T14:15:38,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c205332c8d9a4dd0bf0a5ed581785f4b 2024-11-07T14:15:38,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/6734c62fe83e46ff9d2ffa79a168ee6b is 50, key is test_row_0/C:col10/1730988936960/Put/seqid=0 2024-11-07T14:15:38,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742255_1431 (size=12301) 2024-11-07T14:15:38,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/6734c62fe83e46ff9d2ffa79a168ee6b 2024-11-07T14:15:38,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/d17f5145b0c64b5a99caad241cceaa16 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/d17f5145b0c64b5a99caad241cceaa16 2024-11-07T14:15:38,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/d17f5145b0c64b5a99caad241cceaa16, entries=200, sequenceid=291, filesize=39.0 K 2024-11-07T14:15:38,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/c205332c8d9a4dd0bf0a5ed581785f4b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c205332c8d9a4dd0bf0a5ed581785f4b 2024-11-07T14:15:38,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c205332c8d9a4dd0bf0a5ed581785f4b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-07T14:15:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/6734c62fe83e46ff9d2ffa79a168ee6b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/6734c62fe83e46ff9d2ffa79a168ee6b 2024-11-07T14:15:38,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/6734c62fe83e46ff9d2ffa79a168ee6b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-07T14:15:38,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 186148ebfb7e3f0985eb6829e86c465f in 897ms, sequenceid=291, compaction requested=false 2024-11-07T14:15:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:38,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T14:15:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:38,548 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:15:38,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107342a532a09e64b918710ab11c5d70f8f_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988937665/Put/seqid=0 2024-11-07T14:15:38,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742256_1432 (size=12454) 2024-11-07T14:15:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:38,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988998864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988998866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988998866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988998883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T14:15:38,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:38,967 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107342a532a09e64b918710ab11c5d70f8f_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107342a532a09e64b918710ab11c5d70f8f_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:38,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/b27ab61ba60e40b1a57dc82cdf26d133, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:38,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/b27ab61ba60e40b1a57dc82cdf26d133 is 175, key is test_row_0/A:col10/1730988937665/Put/seqid=0 2024-11-07T14:15:38,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742257_1433 (size=31255) 2024-11-07T14:15:38,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988998982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988998992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988998992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988998993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988999192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988999202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:39,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988999202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:39,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988999203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,372 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/b27ab61ba60e40b1a57dc82cdf26d133 2024-11-07T14:15:39,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/e1a1fd62fa49446990763e87a4870bd3 is 50, key is test_row_0/B:col10/1730988937665/Put/seqid=0 2024-11-07T14:15:39,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742258_1434 (size=12301) 2024-11-07T14:15:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730988999502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730988999512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730988999522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730988999522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:39,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/e1a1fd62fa49446990763e87a4870bd3 2024-11-07T14:15:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/caa1d83d1ee54104a9f47043ff72721b is 50, key is test_row_0/C:col10/1730988937665/Put/seqid=0 2024-11-07T14:15:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742259_1435 (size=12301) 2024-11-07T14:15:39,812 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/caa1d83d1ee54104a9f47043ff72721b 2024-11-07T14:15:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/b27ab61ba60e40b1a57dc82cdf26d133 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/b27ab61ba60e40b1a57dc82cdf26d133 2024-11-07T14:15:39,823 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/b27ab61ba60e40b1a57dc82cdf26d133, entries=150, sequenceid=309, filesize=30.5 K 2024-11-07T14:15:39,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/e1a1fd62fa49446990763e87a4870bd3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e1a1fd62fa49446990763e87a4870bd3 2024-11-07T14:15:39,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e1a1fd62fa49446990763e87a4870bd3, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T14:15:39,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/caa1d83d1ee54104a9f47043ff72721b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/caa1d83d1ee54104a9f47043ff72721b 2024-11-07T14:15:39,842 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/caa1d83d1ee54104a9f47043ff72721b, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T14:15:39,844 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 186148ebfb7e3f0985eb6829e86c465f in 1296ms, sequenceid=309, compaction requested=true 2024-11-07T14:15:39,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:39,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:39,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-07T14:15:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-07T14:15:39,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-07T14:15:39,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-07T14:15:39,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.0770 sec 2024-11-07T14:15:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T14:15:39,925 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:39,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-07T14:15:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T14:15:39,928 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:39,929 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:39,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:40,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:40,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T14:15:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T14:15:40,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110743bc3640877e4c838bc7301f22ae412b_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988940016/Put/seqid=0 2024-11-07T14:15:40,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742260_1436 (size=12454) 2024-11-07T14:15:40,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:40,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:40,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730989000087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730989000087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730989000089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730989000089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730989000212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730989000212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730989000213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730989000214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:40,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:40,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T14:15:40,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730989000422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730989000422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730989000432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730989000434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,459 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:40,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110743bc3640877e4c838bc7301f22ae412b_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743bc3640877e4c838bc7301f22ae412b_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:40,467 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/bb44f588cb25401497191ff76be60206, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:40,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/bb44f588cb25401497191ff76be60206 is 175, key is test_row_0/A:col10/1730988940016/Put/seqid=0 2024-11-07T14:15:40,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742261_1437 (size=31255) 2024-11-07T14:15:40,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T14:15:40,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730989000732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730989000742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730989000743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730989000752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:40,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:40,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:40,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/bb44f588cb25401497191ff76be60206 2024-11-07T14:15:40,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/8ee4f96d4afc4c48a13ddecf9a4b174f is 50, key is test_row_0/B:col10/1730988940016/Put/seqid=0 2024-11-07T14:15:40,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742262_1438 (size=12301) 2024-11-07T14:15:40,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/8ee4f96d4afc4c48a13ddecf9a4b174f 2024-11-07T14:15:40,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/79439f606fea4bff918b58cde2aff4af is 50, key is test_row_0/C:col10/1730988940016/Put/seqid=0 2024-11-07T14:15:41,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742263_1439 (size=12301) 2024-11-07T14:15:41,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T14:15:41,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:41,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:41,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730989001253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:41,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730989001253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730989001255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730989001262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:41,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:41,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:41,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/79439f606fea4bff918b58cde2aff4af 2024-11-07T14:15:41,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/bb44f588cb25401497191ff76be60206 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/bb44f588cb25401497191ff76be60206 2024-11-07T14:15:41,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/bb44f588cb25401497191ff76be60206, entries=150, sequenceid=328, filesize=30.5 K 2024-11-07T14:15:41,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/8ee4f96d4afc4c48a13ddecf9a4b174f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/8ee4f96d4afc4c48a13ddecf9a4b174f 2024-11-07T14:15:41,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/8ee4f96d4afc4c48a13ddecf9a4b174f, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T14:15:41,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/79439f606fea4bff918b58cde2aff4af as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/79439f606fea4bff918b58cde2aff4af 2024-11-07T14:15:41,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:41,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/79439f606fea4bff918b58cde2aff4af, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T14:15:41,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:41,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 186148ebfb7e3f0985eb6829e86c465f in 1503ms, sequenceid=328, compaction requested=true 2024-11-07T14:15:41,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:41,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:41,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:41,523 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:41,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:41,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:41,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:41,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:15:41,525 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:41,529 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:41,529 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/A is initiating minor compaction (all files) 2024-11-07T14:15:41,529 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/A in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,530 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/19d1e9c1e65f4ecea021791319011455, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/d17f5145b0c64b5a99caad241cceaa16, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/b27ab61ba60e40b1a57dc82cdf26d133, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/bb44f588cb25401497191ff76be60206] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=131.2 K 2024-11-07T14:15:41,530 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,530 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/19d1e9c1e65f4ecea021791319011455, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/d17f5145b0c64b5a99caad241cceaa16, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/b27ab61ba60e40b1a57dc82cdf26d133, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/bb44f588cb25401497191ff76be60206] 2024-11-07T14:15:41,532 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d1e9c1e65f4ecea021791319011455, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988936534 2024-11-07T14:15:41,539 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d17f5145b0c64b5a99caad241cceaa16, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730988936948 2024-11-07T14:15:41,542 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:41,542 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/B is initiating minor compaction (all files) 2024-11-07T14:15:41,542 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/B in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,542 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/5c3d38b76cdc4da2998a997442d7adb0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c205332c8d9a4dd0bf0a5ed581785f4b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e1a1fd62fa49446990763e87a4870bd3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/8ee4f96d4afc4c48a13ddecf9a4b174f] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=48.7 K 2024-11-07T14:15:41,542 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27ab61ba60e40b1a57dc82cdf26d133, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988937657 2024-11-07T14:15:41,544 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c3d38b76cdc4da2998a997442d7adb0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988936534 2024-11-07T14:15:41,544 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb44f588cb25401497191ff76be60206, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988938828 2024-11-07T14:15:41,546 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c205332c8d9a4dd0bf0a5ed581785f4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730988936960 2024-11-07T14:15:41,551 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a1fd62fa49446990763e87a4870bd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988937657 2024-11-07T14:15:41,553 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ee4f96d4afc4c48a13ddecf9a4b174f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988938828 2024-11-07T14:15:41,567 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:41,570 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107bf1c03a0ea2343a4949ea90a6ad0a26d_186148ebfb7e3f0985eb6829e86c465f store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:41,572 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#B#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:41,572 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/a8b132dbc46d40ab8ea7c1ac0ee71fdb is 50, key is test_row_0/B:col10/1730988940016/Put/seqid=0 2024-11-07T14:15:41,573 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107bf1c03a0ea2343a4949ea90a6ad0a26d_186148ebfb7e3f0985eb6829e86c465f, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:41,573 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bf1c03a0ea2343a4949ea90a6ad0a26d_186148ebfb7e3f0985eb6829e86c465f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:41,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742264_1440 (size=4469) 2024-11-07T14:15:41,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742265_1441 (size=13085) 2024-11-07T14:15:41,595 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/a8b132dbc46d40ab8ea7c1ac0ee71fdb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a8b132dbc46d40ab8ea7c1ac0ee71fdb 2024-11-07T14:15:41,604 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/B of 186148ebfb7e3f0985eb6829e86c465f into a8b132dbc46d40ab8ea7c1ac0ee71fdb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:41,604 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:41,604 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/B, priority=12, startTime=1730988941523; duration=0sec 2024-11-07T14:15:41,604 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:41,604 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:B 2024-11-07T14:15:41,604 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:41,606 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:41,606 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/C is initiating minor compaction (all files) 2024-11-07T14:15:41,606 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/C in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,607 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/03ccc84f5e734a7d982641d5a9631b77, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/6734c62fe83e46ff9d2ffa79a168ee6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/caa1d83d1ee54104a9f47043ff72721b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/79439f606fea4bff918b58cde2aff4af] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=48.7 K 2024-11-07T14:15:41,607 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ccc84f5e734a7d982641d5a9631b77, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730988936534 2024-11-07T14:15:41,607 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6734c62fe83e46ff9d2ffa79a168ee6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730988936960 2024-11-07T14:15:41,607 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting caa1d83d1ee54104a9f47043ff72721b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730988937657 2024-11-07T14:15:41,608 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 79439f606fea4bff918b58cde2aff4af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988938828 2024-11-07T14:15:41,636 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#C#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:41,636 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/b01cdff250224a788bc076023e5bb898 is 50, key is test_row_0/C:col10/1730988940016/Put/seqid=0 2024-11-07T14:15:41,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T14:15:41,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:41,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742266_1442 (size=13085) 2024-11-07T14:15:41,679 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:15:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d067c72ade74427095a34c3e5627923d_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988940083/Put/seqid=0 2024-11-07T14:15:41,693 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/b01cdff250224a788bc076023e5bb898 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/b01cdff250224a788bc076023e5bb898 2024-11-07T14:15:41,701 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/C of 186148ebfb7e3f0985eb6829e86c465f into b01cdff250224a788bc076023e5bb898(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:41,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:41,701 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/C, priority=12, startTime=1730988941523; duration=0sec 2024-11-07T14:15:41,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:41,701 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:C 2024-11-07T14:15:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742267_1443 (size=12454) 2024-11-07T14:15:41,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:41,733 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d067c72ade74427095a34c3e5627923d_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d067c72ade74427095a34c3e5627923d_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/292a7d409cbb4356aaa3028b8ff707d5, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:41,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/292a7d409cbb4356aaa3028b8ff707d5 is 175, key is test_row_0/A:col10/1730988940083/Put/seqid=0 2024-11-07T14:15:41,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742268_1444 (size=31255) 2024-11-07T14:15:41,986 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#A#compaction#373 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:41,987 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/a1c4f04b3a6f4164aa6895afb172bb90 is 175, key is test_row_0/A:col10/1730988940016/Put/seqid=0 2024-11-07T14:15:41,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742269_1445 (size=32039) 2024-11-07T14:15:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T14:15:42,153 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/292a7d409cbb4356aaa3028b8ff707d5 2024-11-07T14:15:42,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/bdd7fdd40ebd46268d62407977c9fba8 is 50, key is test_row_0/B:col10/1730988940083/Put/seqid=0 2024-11-07T14:15:42,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742270_1446 (size=12301) 2024-11-07T14:15:42,198 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/bdd7fdd40ebd46268d62407977c9fba8 2024-11-07T14:15:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/9754bcd25b26464290ed2040619a62b9 is 50, key is test_row_0/C:col10/1730988940083/Put/seqid=0 2024-11-07T14:15:42,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742271_1447 (size=12301) 2024-11-07T14:15:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:42,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. as already flushing 2024-11-07T14:15:42,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730989002316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730989002318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,338 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-07T14:15:42,338 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,338 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,338 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:42,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730989002327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730989002329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,344 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-11-07T14:15:42,344 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,344 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,344 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,344 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-07T14:15:42,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:42,344 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,344 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,345 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:42,353 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-11-07T14:15:42,353 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,353 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,354 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:42,356 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-07T14:15:42,356 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,356 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,356 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:42,397 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/a1c4f04b3a6f4164aa6895afb172bb90 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a1c4f04b3a6f4164aa6895afb172bb90 2024-11-07T14:15:42,402 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/A of 186148ebfb7e3f0985eb6829e86c465f into a1c4f04b3a6f4164aa6895afb172bb90(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:42,402 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:42,402 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/A, priority=12, startTime=1730988941523; duration=0sec 2024-11-07T14:15:42,402 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:42,402 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:A 2024-11-07T14:15:42,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730989002442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730989002442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52414 deadline: 1730989002452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52416 deadline: 1730989002452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,635 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/9754bcd25b26464290ed2040619a62b9 2024-11-07T14:15:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/292a7d409cbb4356aaa3028b8ff707d5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/292a7d409cbb4356aaa3028b8ff707d5 2024-11-07T14:15:42,644 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/292a7d409cbb4356aaa3028b8ff707d5, entries=150, sequenceid=347, filesize=30.5 K 2024-11-07T14:15:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/bdd7fdd40ebd46268d62407977c9fba8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/bdd7fdd40ebd46268d62407977c9fba8 2024-11-07T14:15:42,648 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/bdd7fdd40ebd46268d62407977c9fba8, entries=150, sequenceid=347, filesize=12.0 K 2024-11-07T14:15:42,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/9754bcd25b26464290ed2040619a62b9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9754bcd25b26464290ed2040619a62b9 2024-11-07T14:15:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52398 deadline: 1730989002652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52380 deadline: 1730989002652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:15:42,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9754bcd25b26464290ed2040619a62b9, entries=150, sequenceid=347, filesize=12.0 K 2024-11-07T14:15:42,655 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 186148ebfb7e3f0985eb6829e86c465f in 976ms, sequenceid=347, compaction requested=false 2024-11-07T14:15:42,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:42,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:42,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-07T14:15:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-07T14:15:42,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-07T14:15:42,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7280 sec 2024-11-07T14:15:42,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.7340 sec 2024-11-07T14:15:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:42,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:42,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T14:15:42,664 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-07T14:15:42,664 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-07T14:15:42,664 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,664 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,664 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,664 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,664 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,664 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:42,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:42,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107271e9ee1d9534e9a981ca72acae412a3_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988942662/Put/seqid=0 2024-11-07T14:15:42,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742272_1448 (size=12454) 2024-11-07T14:15:42,966 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-07T14:15:42,966 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-11-07T14:15:42,966 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,966 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:42,967 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,967 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,967 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,967 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:42,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:42,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:43,074 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:43,083 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107271e9ee1d9534e9a981ca72acae412a3_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107271e9ee1d9534e9a981ca72acae412a3_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:43,084 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/dcbc7172553f46338c506f5b6897dfe3, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:43,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/dcbc7172553f46338c506f5b6897dfe3 is 175, key is test_row_0/A:col10/1730988942662/Put/seqid=0 2024-11-07T14:15:43,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742273_1449 (size=31255) 2024-11-07T14:15:43,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/dcbc7172553f46338c506f5b6897dfe3 2024-11-07T14:15:43,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/65f4cbbd67fd474c833dc6a74f5c16d0 is 50, key is test_row_0/B:col10/1730988942662/Put/seqid=0 2024-11-07T14:15:43,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742274_1450 (size=12301) 2024-11-07T14:15:43,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/65f4cbbd67fd474c833dc6a74f5c16d0 2024-11-07T14:15:43,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/658a61a82a5145d9b18ff56566f07237 is 50, key is test_row_0/C:col10/1730988942662/Put/seqid=0 2024-11-07T14:15:43,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742275_1451 (size=12301) 2024-11-07T14:15:43,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/658a61a82a5145d9b18ff56566f07237 2024-11-07T14:15:43,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/dcbc7172553f46338c506f5b6897dfe3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/dcbc7172553f46338c506f5b6897dfe3 2024-11-07T14:15:43,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/dcbc7172553f46338c506f5b6897dfe3, entries=150, sequenceid=369, filesize=30.5 K 2024-11-07T14:15:43,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/65f4cbbd67fd474c833dc6a74f5c16d0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/65f4cbbd67fd474c833dc6a74f5c16d0 2024-11-07T14:15:43,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/65f4cbbd67fd474c833dc6a74f5c16d0, entries=150, sequenceid=369, filesize=12.0 K 2024-11-07T14:15:43,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/658a61a82a5145d9b18ff56566f07237 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/658a61a82a5145d9b18ff56566f07237 2024-11-07T14:15:43,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/658a61a82a5145d9b18ff56566f07237, entries=150, sequenceid=369, filesize=12.0 K 2024-11-07T14:15:43,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for 186148ebfb7e3f0985eb6829e86c465f in 1273ms, sequenceid=369, compaction requested=true 2024-11-07T14:15:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:43,938 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 186148ebfb7e3f0985eb6829e86c465f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:43,938 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/A is initiating minor compaction (all files) 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/B is initiating minor compaction (all files) 2024-11-07T14:15:43,939 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/A in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:43,939 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/B in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:43,939 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a8b132dbc46d40ab8ea7c1ac0ee71fdb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/bdd7fdd40ebd46268d62407977c9fba8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/65f4cbbd67fd474c833dc6a74f5c16d0] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=36.8 K 2024-11-07T14:15:43,939 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a1c4f04b3a6f4164aa6895afb172bb90, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/292a7d409cbb4356aaa3028b8ff707d5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/dcbc7172553f46338c506f5b6897dfe3] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=92.3 K 2024-11-07T14:15:43,939 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a1c4f04b3a6f4164aa6895afb172bb90, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/292a7d409cbb4356aaa3028b8ff707d5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/dcbc7172553f46338c506f5b6897dfe3] 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting a8b132dbc46d40ab8ea7c1ac0ee71fdb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988938828 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1c4f04b3a6f4164aa6895afb172bb90, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988938828 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting bdd7fdd40ebd46268d62407977c9fba8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1730988940082 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 292a7d409cbb4356aaa3028b8ff707d5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1730988940082 2024-11-07T14:15:43,939 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 65f4cbbd67fd474c833dc6a74f5c16d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730988942316 2024-11-07T14:15:43,940 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcbc7172553f46338c506f5b6897dfe3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730988942316 2024-11-07T14:15:43,947 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:43,947 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:43,947 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/719a34cd15804f49be7e53077ec547d0 is 50, key is test_row_0/B:col10/1730988942662/Put/seqid=0 2024-11-07T14:15:43,953 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110779805a1e6bd442d8895cbfff43dbf680_186148ebfb7e3f0985eb6829e86c465f store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:43,957 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110779805a1e6bd442d8895cbfff43dbf680_186148ebfb7e3f0985eb6829e86c465f, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:43,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742276_1452 (size=13187) 2024-11-07T14:15:43,957 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110779805a1e6bd442d8895cbfff43dbf680_186148ebfb7e3f0985eb6829e86c465f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:43,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742277_1453 (size=4469) 2024-11-07T14:15:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T14:15:44,082 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:44,362 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/719a34cd15804f49be7e53077ec547d0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/719a34cd15804f49be7e53077ec547d0 2024-11-07T14:15:44,364 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#A#compaction#383 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:44,365 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c2d2e05901494c61b62b1d482966af1f is 175, key is test_row_0/A:col10/1730988942662/Put/seqid=0 2024-11-07T14:15:44,367 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/B of 186148ebfb7e3f0985eb6829e86c465f into 719a34cd15804f49be7e53077ec547d0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:44,367 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:44,367 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/B, priority=13, startTime=1730988943938; duration=0sec 2024-11-07T14:15:44,368 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:44,368 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:B 2024-11-07T14:15:44,368 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:44,369 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:44,369 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 186148ebfb7e3f0985eb6829e86c465f/C is initiating minor compaction (all files) 2024-11-07T14:15:44,369 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 186148ebfb7e3f0985eb6829e86c465f/C in TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:44,369 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/b01cdff250224a788bc076023e5bb898, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9754bcd25b26464290ed2040619a62b9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/658a61a82a5145d9b18ff56566f07237] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp, totalSize=36.8 K 2024-11-07T14:15:44,369 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b01cdff250224a788bc076023e5bb898, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988938828 2024-11-07T14:15:44,370 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9754bcd25b26464290ed2040619a62b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1730988940082 2024-11-07T14:15:44,370 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 658a61a82a5145d9b18ff56566f07237, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730988942316 2024-11-07T14:15:44,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742278_1454 (size=32141) 2024-11-07T14:15:44,378 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 186148ebfb7e3f0985eb6829e86c465f#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:44,378 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/55f33bc4e8a244f497c01e6f8ead98cc is 50, key is test_row_0/C:col10/1730988942662/Put/seqid=0 2024-11-07T14:15:44,379 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/c2d2e05901494c61b62b1d482966af1f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c2d2e05901494c61b62b1d482966af1f 2024-11-07T14:15:44,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742279_1455 (size=13187) 2024-11-07T14:15:44,391 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/A of 186148ebfb7e3f0985eb6829e86c465f into c2d2e05901494c61b62b1d482966af1f(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:44,391 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:44,391 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/A, priority=13, startTime=1730988943938; duration=0sec 2024-11-07T14:15:44,391 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:44,391 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:A 2024-11-07T14:15:44,398 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/55f33bc4e8a244f497c01e6f8ead98cc as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/55f33bc4e8a244f497c01e6f8ead98cc 2024-11-07T14:15:44,404 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 186148ebfb7e3f0985eb6829e86c465f/C of 186148ebfb7e3f0985eb6829e86c465f into 55f33bc4e8a244f497c01e6f8ead98cc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:44,404 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 186148ebfb7e3f0985eb6829e86c465f: 2024-11-07T14:15:44,404 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., storeName=186148ebfb7e3f0985eb6829e86c465f/C, priority=13, startTime=1730988943938; duration=0sec 2024-11-07T14:15:44,404 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:44,404 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 186148ebfb7e3f0985eb6829e86c465f:C 2024-11-07T14:15:45,077 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:15:45,695 INFO [Thread-1581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1581. 2024-11-07T14:15:45,695 DEBUG [Thread-1581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:15:45,695 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:45,695 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:45,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1271 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3812 rows 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1280 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3840 rows 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1283 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3847 rows 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1282 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3846 rows 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1279 2024-11-07T14:15:45,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3837 rows 2024-11-07T14:15:45,697 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T14:15:45,697 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:15:45,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:45,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:45,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:15:45,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T14:15:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T14:15:45,703 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988945703"}]},"ts":"1730988945703"} 2024-11-07T14:15:45,704 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T14:15:45,705 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T14:15:45,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:15:45,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, UNASSIGN}] 2024-11-07T14:15:45,707 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, UNASSIGN 2024-11-07T14:15:45,707 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=186148ebfb7e3f0985eb6829e86c465f, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:45,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, UNASSIGN because future has completed 2024-11-07T14:15:45,709 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:15:45,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:15:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T14:15:45,861 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:45,861 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:15:45,861 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 186148ebfb7e3f0985eb6829e86c465f, disabling compactions & flushes 2024-11-07T14:15:45,861 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:45,861 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:45,861 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. after waiting 0 ms 2024-11-07T14:15:45,861 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:45,861 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 186148ebfb7e3f0985eb6829e86c465f 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T14:15:45,868 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110748ef128f27db4ac3a5dce3813d353936_186148ebfb7e3f0985eb6829e86c465f is 50, key is test_row_0/A:col10/1730988942963/Put/seqid=0 2024-11-07T14:15:45,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742280_1456 (size=9914) 2024-11-07T14:15:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T14:15:46,273 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:46,283 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110748ef128f27db4ac3a5dce3813d353936_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110748ef128f27db4ac3a5dce3813d353936_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:46,284 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/a46aea228d6f449ea28ba491fd708511, store: [table=TestAcidGuarantees family=A region=186148ebfb7e3f0985eb6829e86c465f] 2024-11-07T14:15:46,285 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/a46aea228d6f449ea28ba491fd708511 is 175, key is test_row_0/A:col10/1730988942963/Put/seqid=0 2024-11-07T14:15:46,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742281_1457 (size=22561) 2024-11-07T14:15:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T14:15:46,689 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/a46aea228d6f449ea28ba491fd708511 2024-11-07T14:15:46,695 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/e7be40e2b34149898d0a6bb39d6a4784 is 50, key is test_row_0/B:col10/1730988942963/Put/seqid=0 2024-11-07T14:15:46,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742282_1458 (size=9857) 2024-11-07T14:15:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T14:15:47,100 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/e7be40e2b34149898d0a6bb39d6a4784 2024-11-07T14:15:47,112 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/25d0153918e949c593fee69fa07755ea is 50, key is test_row_0/C:col10/1730988942963/Put/seqid=0 2024-11-07T14:15:47,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742283_1459 (size=9857) 2024-11-07T14:15:47,517 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/25d0153918e949c593fee69fa07755ea 2024-11-07T14:15:47,521 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/A/a46aea228d6f449ea28ba491fd708511 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a46aea228d6f449ea28ba491fd708511 2024-11-07T14:15:47,524 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a46aea228d6f449ea28ba491fd708511, entries=100, sequenceid=378, filesize=22.0 K 2024-11-07T14:15:47,525 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/B/e7be40e2b34149898d0a6bb39d6a4784 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e7be40e2b34149898d0a6bb39d6a4784 2024-11-07T14:15:47,528 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e7be40e2b34149898d0a6bb39d6a4784, entries=100, sequenceid=378, filesize=9.6 K 2024-11-07T14:15:47,528 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/.tmp/C/25d0153918e949c593fee69fa07755ea as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/25d0153918e949c593fee69fa07755ea 2024-11-07T14:15:47,532 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/25d0153918e949c593fee69fa07755ea, entries=100, sequenceid=378, filesize=9.6 K 2024-11-07T14:15:47,533 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 186148ebfb7e3f0985eb6829e86c465f in 1672ms, sequenceid=378, compaction requested=false 2024-11-07T14:15:47,533 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/30dae00288e7474796992a0624693e55, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/83b1575ed6174c3395554cd4a5df2edf, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c877c296e0b844c99fd7a6e1b02960e8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/468b57c3566c436281985cc177295fd9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/778a65ff9f604f8b897979b7694ffeb2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c6a769ad3b664774b5e871a9557c22b0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/2b8332a7b8e84dc1a653ef87b95a26c7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c0af3b5e51fe463a8b65f671ce945ea9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/f14dbd5b9854461496ec25660083aa20, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5950715a3bab4eb180814e2949fb35a4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c934effe2d3a4360b200509f555b5bc8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/33334af101584e63a1b863f533e0203d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/7b08d3f31a364beca0bf308eeca4613d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/4523f16379ba4cb6aa984ed805a5a510, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/e54b959314dc4c4aaad2fbaec5dcf682, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/60737df2017c4f2db4aa3c1d981e9aee, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/773a3a9d8b9f4922912dea7f1849d18e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/df14fe58cc7a436db463459f97f7c8cc, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5ae3d594698e4af0bc37a2e68e3063b8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/19d1e9c1e65f4ecea021791319011455, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/d17f5145b0c64b5a99caad241cceaa16, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/b27ab61ba60e40b1a57dc82cdf26d133, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a1c4f04b3a6f4164aa6895afb172bb90, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/bb44f588cb25401497191ff76be60206, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/292a7d409cbb4356aaa3028b8ff707d5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/dcbc7172553f46338c506f5b6897dfe3] to archive 2024-11-07T14:15:47,534 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:15:47,536 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/30dae00288e7474796992a0624693e55 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/30dae00288e7474796992a0624693e55 2024-11-07T14:15:47,537 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/83b1575ed6174c3395554cd4a5df2edf to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/83b1575ed6174c3395554cd4a5df2edf 2024-11-07T14:15:47,538 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c877c296e0b844c99fd7a6e1b02960e8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c877c296e0b844c99fd7a6e1b02960e8 2024-11-07T14:15:47,539 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/468b57c3566c436281985cc177295fd9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/468b57c3566c436281985cc177295fd9 2024-11-07T14:15:47,541 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/778a65ff9f604f8b897979b7694ffeb2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/778a65ff9f604f8b897979b7694ffeb2 2024-11-07T14:15:47,542 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c6a769ad3b664774b5e871a9557c22b0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c6a769ad3b664774b5e871a9557c22b0 2024-11-07T14:15:47,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/2b8332a7b8e84dc1a653ef87b95a26c7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/2b8332a7b8e84dc1a653ef87b95a26c7 2024-11-07T14:15:47,544 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c0af3b5e51fe463a8b65f671ce945ea9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c0af3b5e51fe463a8b65f671ce945ea9 2024-11-07T14:15:47,544 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/f14dbd5b9854461496ec25660083aa20 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/f14dbd5b9854461496ec25660083aa20 2024-11-07T14:15:47,545 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5950715a3bab4eb180814e2949fb35a4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5950715a3bab4eb180814e2949fb35a4 2024-11-07T14:15:47,546 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c934effe2d3a4360b200509f555b5bc8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c934effe2d3a4360b200509f555b5bc8 2024-11-07T14:15:47,547 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/33334af101584e63a1b863f533e0203d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/33334af101584e63a1b863f533e0203d 2024-11-07T14:15:47,548 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/7b08d3f31a364beca0bf308eeca4613d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/7b08d3f31a364beca0bf308eeca4613d 2024-11-07T14:15:47,549 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/4523f16379ba4cb6aa984ed805a5a510 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/4523f16379ba4cb6aa984ed805a5a510 2024-11-07T14:15:47,550 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/e54b959314dc4c4aaad2fbaec5dcf682 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/e54b959314dc4c4aaad2fbaec5dcf682 2024-11-07T14:15:47,551 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/60737df2017c4f2db4aa3c1d981e9aee to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/60737df2017c4f2db4aa3c1d981e9aee 2024-11-07T14:15:47,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/773a3a9d8b9f4922912dea7f1849d18e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/773a3a9d8b9f4922912dea7f1849d18e 2024-11-07T14:15:47,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/df14fe58cc7a436db463459f97f7c8cc to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/df14fe58cc7a436db463459f97f7c8cc 2024-11-07T14:15:47,553 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5ae3d594698e4af0bc37a2e68e3063b8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/5ae3d594698e4af0bc37a2e68e3063b8 2024-11-07T14:15:47,554 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/19d1e9c1e65f4ecea021791319011455 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/19d1e9c1e65f4ecea021791319011455 2024-11-07T14:15:47,555 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/d17f5145b0c64b5a99caad241cceaa16 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/d17f5145b0c64b5a99caad241cceaa16 2024-11-07T14:15:47,556 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/b27ab61ba60e40b1a57dc82cdf26d133 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/b27ab61ba60e40b1a57dc82cdf26d133 2024-11-07T14:15:47,557 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a1c4f04b3a6f4164aa6895afb172bb90 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a1c4f04b3a6f4164aa6895afb172bb90 2024-11-07T14:15:47,558 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/bb44f588cb25401497191ff76be60206 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/bb44f588cb25401497191ff76be60206 2024-11-07T14:15:47,559 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/292a7d409cbb4356aaa3028b8ff707d5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/292a7d409cbb4356aaa3028b8ff707d5 2024-11-07T14:15:47,560 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/dcbc7172553f46338c506f5b6897dfe3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/dcbc7172553f46338c506f5b6897dfe3 2024-11-07T14:15:47,561 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72bde7a0cbb94adf9f1c845ac51748f6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c845ffecff0544b18e07f7aa6d096df7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c9685e8ced7d4789b945eb5991c0b2e1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/28fc7ae0d8714e9993fdab75fcc58d9b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/02a0ce8ad6f7425380d7657613b37443, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/651a799f99a7485e99cf095fc9817535, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c483b2fd065749dbb1bb089c32fdff35, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/cf2d86234ac74b36b6dd060acbcff5eb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72dc388392ec40d8959c9b659045ce9b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/f878669e120b4d47a88764acde3d92e4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33538fe7e6b345f294d7756671759b49, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/3bbc3a8899e14ac1a851b801241e0138, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/b51f34d171224c92a4b9594e7b49f17d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/1437b30e50ea456d8e4d013eb8199fa2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/58df401642c8471888e14606d20ccfac, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/4e058a900103421db085d2a29c763c7c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33d3af56323a4e8e8ee4b02531291ac9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/5c3d38b76cdc4da2998a997442d7adb0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/46980c29809d4a8f97208aa5e727a3e8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c205332c8d9a4dd0bf0a5ed581785f4b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e1a1fd62fa49446990763e87a4870bd3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a8b132dbc46d40ab8ea7c1ac0ee71fdb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/8ee4f96d4afc4c48a13ddecf9a4b174f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/bdd7fdd40ebd46268d62407977c9fba8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/65f4cbbd67fd474c833dc6a74f5c16d0] to archive 2024-11-07T14:15:47,562 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:15:47,563 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72bde7a0cbb94adf9f1c845ac51748f6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72bde7a0cbb94adf9f1c845ac51748f6 2024-11-07T14:15:47,564 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c845ffecff0544b18e07f7aa6d096df7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c845ffecff0544b18e07f7aa6d096df7 2024-11-07T14:15:47,565 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c9685e8ced7d4789b945eb5991c0b2e1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c9685e8ced7d4789b945eb5991c0b2e1 2024-11-07T14:15:47,565 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/28fc7ae0d8714e9993fdab75fcc58d9b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/28fc7ae0d8714e9993fdab75fcc58d9b 2024-11-07T14:15:47,567 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/02a0ce8ad6f7425380d7657613b37443 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/02a0ce8ad6f7425380d7657613b37443 2024-11-07T14:15:47,568 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/651a799f99a7485e99cf095fc9817535 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/651a799f99a7485e99cf095fc9817535 2024-11-07T14:15:47,568 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c483b2fd065749dbb1bb089c32fdff35 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c483b2fd065749dbb1bb089c32fdff35 2024-11-07T14:15:47,570 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/cf2d86234ac74b36b6dd060acbcff5eb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/cf2d86234ac74b36b6dd060acbcff5eb 2024-11-07T14:15:47,571 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a7bb590f8a3b46b2b8a7a2bf4ec0d8c6 2024-11-07T14:15:47,571 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72dc388392ec40d8959c9b659045ce9b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/72dc388392ec40d8959c9b659045ce9b 2024-11-07T14:15:47,572 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/f878669e120b4d47a88764acde3d92e4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/f878669e120b4d47a88764acde3d92e4 2024-11-07T14:15:47,573 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33538fe7e6b345f294d7756671759b49 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33538fe7e6b345f294d7756671759b49 2024-11-07T14:15:47,574 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/3bbc3a8899e14ac1a851b801241e0138 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/3bbc3a8899e14ac1a851b801241e0138 2024-11-07T14:15:47,574 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/b51f34d171224c92a4b9594e7b49f17d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/b51f34d171224c92a4b9594e7b49f17d 2024-11-07T14:15:47,575 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/1437b30e50ea456d8e4d013eb8199fa2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/1437b30e50ea456d8e4d013eb8199fa2 2024-11-07T14:15:47,576 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/58df401642c8471888e14606d20ccfac to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/58df401642c8471888e14606d20ccfac 2024-11-07T14:15:47,577 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/4e058a900103421db085d2a29c763c7c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/4e058a900103421db085d2a29c763c7c 2024-11-07T14:15:47,577 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33d3af56323a4e8e8ee4b02531291ac9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/33d3af56323a4e8e8ee4b02531291ac9 2024-11-07T14:15:47,578 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/5c3d38b76cdc4da2998a997442d7adb0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/5c3d38b76cdc4da2998a997442d7adb0 2024-11-07T14:15:47,579 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/46980c29809d4a8f97208aa5e727a3e8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/46980c29809d4a8f97208aa5e727a3e8 2024-11-07T14:15:47,580 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c205332c8d9a4dd0bf0a5ed581785f4b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/c205332c8d9a4dd0bf0a5ed581785f4b 2024-11-07T14:15:47,581 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e1a1fd62fa49446990763e87a4870bd3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e1a1fd62fa49446990763e87a4870bd3 2024-11-07T14:15:47,581 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a8b132dbc46d40ab8ea7c1ac0ee71fdb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/a8b132dbc46d40ab8ea7c1ac0ee71fdb 2024-11-07T14:15:47,582 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/8ee4f96d4afc4c48a13ddecf9a4b174f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/8ee4f96d4afc4c48a13ddecf9a4b174f 2024-11-07T14:15:47,583 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/bdd7fdd40ebd46268d62407977c9fba8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/bdd7fdd40ebd46268d62407977c9fba8 2024-11-07T14:15:47,584 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/65f4cbbd67fd474c833dc6a74f5c16d0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/65f4cbbd67fd474c833dc6a74f5c16d0 2024-11-07T14:15:47,585 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9f5ffe5c242b4958bbe0960a493e07cd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/43db3ec5401f467aac712eb6de551eaa, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/271bdd73d1f14660adb218583abdec67, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/f2f82fefedb6410fa0f2f5ac3477bcf3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/d08112df6788493fa98e844cc60213bd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e0ec9c3c0a344f18977dc6e2127dfb51, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5144b805e00e4e639cd421932a8c1756, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e05f64a015a949dbb486ebd978c277a0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/a24fa6cd78bf46a5b46784d8efd5dfb9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2788ebc33cf7480fb140cd104d2a7b9f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/86ad32c2c51a46e58a6a45d12f3b368c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e7daff4cdcc34ceb8cfb5e4f39257a91, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5cb327d8524e42d0bf5cf010b8518e18, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/447a586390c2476680132185ee807f00, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/c844064d0cc64dae9aac549f3f958dd9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2779a6246efe4d8ca29c24fe7dfc5bc7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/35fb7e594f94450db34b4596338482b0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/827c759e131c407f90c37b7a2300abe2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/03ccc84f5e734a7d982641d5a9631b77, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/0efbdd7ba40d41fd801a4cc07a8cb0cf, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/6734c62fe83e46ff9d2ffa79a168ee6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/caa1d83d1ee54104a9f47043ff72721b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/b01cdff250224a788bc076023e5bb898, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/79439f606fea4bff918b58cde2aff4af, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9754bcd25b26464290ed2040619a62b9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/658a61a82a5145d9b18ff56566f07237] to archive 2024-11-07T14:15:47,586 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:15:47,587 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9f5ffe5c242b4958bbe0960a493e07cd to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9f5ffe5c242b4958bbe0960a493e07cd 2024-11-07T14:15:47,588 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/43db3ec5401f467aac712eb6de551eaa to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/43db3ec5401f467aac712eb6de551eaa 2024-11-07T14:15:47,588 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/271bdd73d1f14660adb218583abdec67 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/271bdd73d1f14660adb218583abdec67 2024-11-07T14:15:47,589 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/f2f82fefedb6410fa0f2f5ac3477bcf3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/f2f82fefedb6410fa0f2f5ac3477bcf3 2024-11-07T14:15:47,590 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/d08112df6788493fa98e844cc60213bd to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/d08112df6788493fa98e844cc60213bd 2024-11-07T14:15:47,590 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e0ec9c3c0a344f18977dc6e2127dfb51 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e0ec9c3c0a344f18977dc6e2127dfb51 2024-11-07T14:15:47,592 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5144b805e00e4e639cd421932a8c1756 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5144b805e00e4e639cd421932a8c1756 2024-11-07T14:15:47,592 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e05f64a015a949dbb486ebd978c277a0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e05f64a015a949dbb486ebd978c277a0 2024-11-07T14:15:47,593 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/a24fa6cd78bf46a5b46784d8efd5dfb9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/a24fa6cd78bf46a5b46784d8efd5dfb9 2024-11-07T14:15:47,594 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2788ebc33cf7480fb140cd104d2a7b9f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2788ebc33cf7480fb140cd104d2a7b9f 2024-11-07T14:15:47,595 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/86ad32c2c51a46e58a6a45d12f3b368c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/86ad32c2c51a46e58a6a45d12f3b368c 2024-11-07T14:15:47,595 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e7daff4cdcc34ceb8cfb5e4f39257a91 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/e7daff4cdcc34ceb8cfb5e4f39257a91 2024-11-07T14:15:47,596 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5cb327d8524e42d0bf5cf010b8518e18 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/5cb327d8524e42d0bf5cf010b8518e18 2024-11-07T14:15:47,597 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/447a586390c2476680132185ee807f00 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/447a586390c2476680132185ee807f00 2024-11-07T14:15:47,598 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/c844064d0cc64dae9aac549f3f958dd9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/c844064d0cc64dae9aac549f3f958dd9 2024-11-07T14:15:47,598 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2779a6246efe4d8ca29c24fe7dfc5bc7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/2779a6246efe4d8ca29c24fe7dfc5bc7 2024-11-07T14:15:47,599 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/35fb7e594f94450db34b4596338482b0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/35fb7e594f94450db34b4596338482b0 2024-11-07T14:15:47,600 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/827c759e131c407f90c37b7a2300abe2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/827c759e131c407f90c37b7a2300abe2 2024-11-07T14:15:47,601 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/03ccc84f5e734a7d982641d5a9631b77 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/03ccc84f5e734a7d982641d5a9631b77 2024-11-07T14:15:47,601 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/0efbdd7ba40d41fd801a4cc07a8cb0cf to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/0efbdd7ba40d41fd801a4cc07a8cb0cf 2024-11-07T14:15:47,602 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/6734c62fe83e46ff9d2ffa79a168ee6b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/6734c62fe83e46ff9d2ffa79a168ee6b 2024-11-07T14:15:47,603 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/caa1d83d1ee54104a9f47043ff72721b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/caa1d83d1ee54104a9f47043ff72721b 2024-11-07T14:15:47,604 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/b01cdff250224a788bc076023e5bb898 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/b01cdff250224a788bc076023e5bb898 2024-11-07T14:15:47,604 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/79439f606fea4bff918b58cde2aff4af to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/79439f606fea4bff918b58cde2aff4af 2024-11-07T14:15:47,605 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9754bcd25b26464290ed2040619a62b9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/9754bcd25b26464290ed2040619a62b9 2024-11-07T14:15:47,605 DEBUG [StoreCloser-TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/658a61a82a5145d9b18ff56566f07237 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/658a61a82a5145d9b18ff56566f07237 2024-11-07T14:15:47,610 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/recovered.edits/381.seqid, newMaxSeqId=381, maxSeqId=4 2024-11-07T14:15:47,610 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. 2024-11-07T14:15:47,610 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 186148ebfb7e3f0985eb6829e86c465f: Waiting for close lock at 1730988945861Running coprocessor pre-close hooks at 1730988945861Disabling compacts and flushes for region at 1730988945861Disabling writes for close at 1730988945861Obtaining lock to block concurrent updates at 1730988945861Preparing flush snapshotting stores in 186148ebfb7e3f0985eb6829e86c465f at 1730988945861Finished memstore snapshotting TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730988945862 (+1 ms)Flushing stores of TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f. at 1730988945862Flushing 186148ebfb7e3f0985eb6829e86c465f/A: creating writer at 1730988945862Flushing mob file 186148ebfb7e3f0985eb6829e86c465f/A: appending metadata at 1730988945867 (+5 ms)Flushing mob file 186148ebfb7e3f0985eb6829e86c465f/A: closing flushed file at 1730988945867Flushing 186148ebfb7e3f0985eb6829e86c465f/A: appending metadata at 1730988946284 (+417 ms)Flushing 186148ebfb7e3f0985eb6829e86c465f/A: closing flushed file at 1730988946284Flushing 186148ebfb7e3f0985eb6829e86c465f/B: creating writer at 1730988946693 (+409 ms)Flushing 186148ebfb7e3f0985eb6829e86c465f/B: appending metadata at 1730988946695 (+2 ms)Flushing 186148ebfb7e3f0985eb6829e86c465f/B: closing flushed file at 1730988946695Flushing 186148ebfb7e3f0985eb6829e86c465f/C: creating writer at 1730988947108 (+413 ms)Flushing 186148ebfb7e3f0985eb6829e86c465f/C: appending metadata at 1730988947111 (+3 ms)Flushing 186148ebfb7e3f0985eb6829e86c465f/C: closing flushed file at 1730988947111Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@522b6b11: reopening flushed file at 1730988947520 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a7a9a88: reopening flushed file at 1730988947525 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11981524: reopening flushed file at 1730988947528 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 186148ebfb7e3f0985eb6829e86c465f in 1672ms, sequenceid=378, compaction requested=false at 1730988947533 (+5 ms)Writing region close event to WAL at 1730988947607 (+74 ms)Running coprocessor post-close hooks at 1730988947610 (+3 ms)Closed at 1730988947610 2024-11-07T14:15:47,612 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,613 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=186148ebfb7e3f0985eb6829e86c465f, regionState=CLOSED 2024-11-07T14:15:47,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:15:47,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-07T14:15:47,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 186148ebfb7e3f0985eb6829e86c465f, server=b7cc9d764de6,37737,1730988826985 in 1.9050 sec 2024-11-07T14:15:47,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-07T14:15:47,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=186148ebfb7e3f0985eb6829e86c465f, UNASSIGN in 1.9100 sec 2024-11-07T14:15:47,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-07T14:15:47,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9130 sec 2024-11-07T14:15:47,620 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988947620"}]},"ts":"1730988947620"} 2024-11-07T14:15:47,621 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T14:15:47,621 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T14:15:47,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9240 sec 2024-11-07T14:15:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T14:15:47,853 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:47,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T14:15:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:47,855 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T14:15:47,856 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:47,858 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,860 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/recovered.edits] 2024-11-07T14:15:47,861 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a46aea228d6f449ea28ba491fd708511 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/a46aea228d6f449ea28ba491fd708511 2024-11-07T14:15:47,862 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c2d2e05901494c61b62b1d482966af1f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/A/c2d2e05901494c61b62b1d482966af1f 2024-11-07T14:15:47,864 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/719a34cd15804f49be7e53077ec547d0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/719a34cd15804f49be7e53077ec547d0 2024-11-07T14:15:47,865 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e7be40e2b34149898d0a6bb39d6a4784 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/B/e7be40e2b34149898d0a6bb39d6a4784 2024-11-07T14:15:47,871 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/25d0153918e949c593fee69fa07755ea to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/25d0153918e949c593fee69fa07755ea 2024-11-07T14:15:47,872 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/55f33bc4e8a244f497c01e6f8ead98cc to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/C/55f33bc4e8a244f497c01e6f8ead98cc 2024-11-07T14:15:47,874 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/recovered.edits/381.seqid to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f/recovered.edits/381.seqid 2024-11-07T14:15:47,874 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,875 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T14:15:47,875 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T14:15:47,875 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T14:15:47,878 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107271e9ee1d9534e9a981ca72acae412a3_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107271e9ee1d9534e9a981ca72acae412a3_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,878 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072eb47917e05645a5bb2d5421c3c3f98f_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072eb47917e05645a5bb2d5421c3c3f98f_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,879 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107342a532a09e64b918710ab11c5d70f8f_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107342a532a09e64b918710ab11c5d70f8f_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073d019db827b147b0ace6a465af3b39ce_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073d019db827b147b0ace6a465af3b39ce_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,881 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743bc3640877e4c838bc7301f22ae412b_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743bc3640877e4c838bc7301f22ae412b_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,882 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110748ef128f27db4ac3a5dce3813d353936_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110748ef128f27db4ac3a5dce3813d353936_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,882 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107557296566eb945ab972bdaecccb7e626_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107557296566eb945ab972bdaecccb7e626_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,883 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110768653f413ddd412a8032691a6bfaa55d_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110768653f413ddd412a8032691a6bfaa55d_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076b85d187ca1e429ea54b0d96bade591b_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076b85d187ca1e429ea54b0d96bade591b_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ce337a753ed4c8db1c8f5c6a794c6c6_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ce337a753ed4c8db1c8f5c6a794c6c6_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a9fa7b556e04a1c8dda02fa32b7af1b_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078a9fa7b556e04a1c8dda02fa32b7af1b_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,886 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078f5341915996417088934c4e346f418a_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078f5341915996417088934c4e346f418a_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,887 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a9cb715e4e1344db8e4e90f5048389f7_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a9cb715e4e1344db8e4e90f5048389f7_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,887 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aaaa5d8b53bd46228f48c3f8ca9099c8_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aaaa5d8b53bd46228f48c3f8ca9099c8_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,888 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b62ee4824ad54098a3e25367308aa233_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b62ee4824ad54098a3e25367308aa233_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,889 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b805683f26cb4745a76021d62c889304_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b805683f26cb4745a76021d62c889304_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,890 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c86c6b9dad254f8b9396deb4d8884f0e_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c86c6b9dad254f8b9396deb4d8884f0e_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,890 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d067c72ade74427095a34c3e5627923d_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d067c72ade74427095a34c3e5627923d_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,891 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e90d60bafe19475084c35c4228a23c86_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e90d60bafe19475084c35c4228a23c86_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,892 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f79ccdc7c50245e3bc8ea244f74898f9_186148ebfb7e3f0985eb6829e86c465f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f79ccdc7c50245e3bc8ea244f74898f9_186148ebfb7e3f0985eb6829e86c465f 2024-11-07T14:15:47,892 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T14:15:47,894 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:47,896 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T14:15:47,898 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T14:15:47,899 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:47,899 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T14:15:47,899 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730988947899"}]},"ts":"9223372036854775807"} 2024-11-07T14:15:47,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T14:15:47,902 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 186148ebfb7e3f0985eb6829e86c465f, NAME => 'TestAcidGuarantees,,1730988919173.186148ebfb7e3f0985eb6829e86c465f.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T14:15:47,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T14:15:47,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730988947902"}]},"ts":"9223372036854775807"} 2024-11-07T14:15:47,904 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T14:15:47,905 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:47,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-07T14:15:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T14:15:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T14:15:47,963 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:47,974 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1469413959_22 at /127.0.0.1:42050 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1809816429_22 at /127.0.0.1:57188 [Waiting for operation #1238] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1809816429_22 at /127.0.0.1:54994 [Waiting for operation #420] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1809816429_22 at /127.0.0.1:55004 [Waiting for operation #417] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1469413959_22 at /127.0.0.1:55012 [Waiting for operation #415] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=566 (was 563) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=591 (was 239) - AvailableMemoryMB LEAK? - 2024-11-07T14:15:47,983 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=566, ProcessCount=11, AvailableMemoryMB=590 2024-11-07T14:15:47,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:15:47,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:15:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T14:15:47,986 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:15:47,986 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:47,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-07T14:15:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T14:15:47,987 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:15:47,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742284_1460 (size=959) 2024-11-07T14:15:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T14:15:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T14:15:48,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 2024-11-07T14:15:48,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742285_1461 (size=53) 2024-11-07T14:15:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T14:15:48,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:15:48,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 758cb99e0ef7c32067829f2f2f90dc44, disabling compactions & flushes 2024-11-07T14:15:48,798 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:48,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:48,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. after waiting 0 ms 2024-11-07T14:15:48,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:48,798 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:48,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 758cb99e0ef7c32067829f2f2f90dc44: Waiting for close lock at 1730988948798Disabling compacts and flushes for region at 1730988948798Disabling writes for close at 1730988948798Writing region close event to WAL at 1730988948798Closed at 1730988948798 2024-11-07T14:15:48,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:15:48,800 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730988948800"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730988948800"}]},"ts":"1730988948800"} 2024-11-07T14:15:48,801 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T14:15:48,802 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:15:48,802 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988948802"}]},"ts":"1730988948802"} 2024-11-07T14:15:48,803 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T14:15:48,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, ASSIGN}] 2024-11-07T14:15:48,805 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, ASSIGN 2024-11-07T14:15:48,805 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, ASSIGN; state=OFFLINE, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=false 2024-11-07T14:15:48,956 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=758cb99e0ef7c32067829f2f2f90dc44, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:48,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, ASSIGN because future has completed 2024-11-07T14:15:48,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:15:49,115 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:49,115 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:15:49,116 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,116 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:15:49,116 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,117 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,118 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,120 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 758cb99e0ef7c32067829f2f2f90dc44 columnFamilyName A 2024-11-07T14:15:49,120 DEBUG [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:49,121 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] regionserver.HStore(327): Store=758cb99e0ef7c32067829f2f2f90dc44/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:49,121 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T14:15:49,123 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 758cb99e0ef7c32067829f2f2f90dc44 columnFamilyName B 2024-11-07T14:15:49,123 DEBUG [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:49,123 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] regionserver.HStore(327): Store=758cb99e0ef7c32067829f2f2f90dc44/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:49,124 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,125 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 758cb99e0ef7c32067829f2f2f90dc44 columnFamilyName C 2024-11-07T14:15:49,125 DEBUG [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:15:49,126 INFO [StoreOpener-758cb99e0ef7c32067829f2f2f90dc44-1 {}] regionserver.HStore(327): Store=758cb99e0ef7c32067829f2f2f90dc44/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:15:49,126 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,127 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,127 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,129 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,129 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,129 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:15:49,131 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,133 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:15:49,134 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 758cb99e0ef7c32067829f2f2f90dc44; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71296815, jitterRate=0.062405332922935486}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:15:49,134 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:49,135 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 758cb99e0ef7c32067829f2f2f90dc44: Running coprocessor pre-open hook at 1730988949117Writing region info on filesystem at 1730988949117Initializing all the Stores at 1730988949118 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988949118Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988949118Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988949118Cleaning up temporary data from old regions at 1730988949129 (+11 ms)Running coprocessor post-open hooks at 1730988949134 (+5 ms)Region opened successfully at 1730988949135 (+1 ms) 2024-11-07T14:15:49,136 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., pid=126, masterSystemTime=1730988949111 2024-11-07T14:15:49,138 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:49,139 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:49,139 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=758cb99e0ef7c32067829f2f2f90dc44, regionState=OPEN, openSeqNum=2, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:49,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:15:49,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-07T14:15:49,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 in 183 msec 2024-11-07T14:15:49,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-07T14:15:49,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, ASSIGN in 338 msec 2024-11-07T14:15:49,144 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:15:49,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988949144"}]},"ts":"1730988949144"} 2024-11-07T14:15:49,146 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T14:15:49,146 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:15:49,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-07T14:15:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-07T14:15:50,132 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:50,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3a0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,134 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5097581c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,136 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c5d2d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,137 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,140 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T14:15:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b7cc9d764de6,40745,1730988826115 2024-11-07T14:15:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@475588b7 2024-11-07T14:15:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T14:15:50,142 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T14:15:50,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c2b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,145 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c24998a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,146 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e6fbc7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57af2b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,148 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2bfb18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,150 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea38460, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ae5a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,152 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d34f6d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,153 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727630f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fe770e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,155 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@823d963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,156 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7efef54a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cc712e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,159 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c43517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,161 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613fc48e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29cadc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,163 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1003efd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,165 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@772bc44b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b7256f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,168 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37846f24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb67001, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@188c1103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,174 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270fee16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,175 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0a381c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f46cad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,177 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1972522e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,179 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1601fbe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d22d375, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:15:50,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:15:50,181 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6134ec28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:15:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:15:50,182 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:15:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b4b9345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:15:50,183 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,183 DEBUG [Thread-2067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,184 DEBUG [Thread-2074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,186 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,190 DEBUG [Thread-2082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,191 DEBUG [Thread-2080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:50,191 DEBUG [Thread-2076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-07T14:15:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T14:15:50,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,196 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,203 DEBUG [Thread-2065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,207 DEBUG [Thread-2071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,212 DEBUG [Thread-2078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:15:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:15:50,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:15:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2] 2024-11-07T14:15:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:50,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:15:50,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/1250fce944b440878e82a2131c8301a7 is 50, key is test_row_0/A:col10/1730988950236/Put/seqid=0 2024-11-07T14:15:50,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742286_1462 (size=12001) 2024-11-07T14:15:50,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/1250fce944b440878e82a2131c8301a7 2024-11-07T14:15:50,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T14:15:50,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989010291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989010292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989010296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989010297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/acdf09be141c40188b308f4b86be9aa1 is 50, key is test_row_0/B:col10/1730988950236/Put/seqid=0 2024-11-07T14:15:50,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989010315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742287_1463 (size=12001) 2024-11-07T14:15:50,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/acdf09be141c40188b308f4b86be9aa1 2024-11-07T14:15:50,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T14:15:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/4079f92164024697979d8579919f8830 is 50, key is test_row_0/C:col10/1730988950236/Put/seqid=0 2024-11-07T14:15:50,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742288_1464 (size=12001) 2024-11-07T14:15:50,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/4079f92164024697979d8579919f8830 2024-11-07T14:15:50,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/1250fce944b440878e82a2131c8301a7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1250fce944b440878e82a2131c8301a7 2024-11-07T14:15:50,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1250fce944b440878e82a2131c8301a7, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T14:15:50,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/acdf09be141c40188b308f4b86be9aa1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/acdf09be141c40188b308f4b86be9aa1 2024-11-07T14:15:50,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/acdf09be141c40188b308f4b86be9aa1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T14:15:50,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/4079f92164024697979d8579919f8830 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/4079f92164024697979d8579919f8830 2024-11-07T14:15:50,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/4079f92164024697979d8579919f8830, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T14:15:50,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 758cb99e0ef7c32067829f2f2f90dc44 in 158ms, sequenceid=14, compaction requested=false 2024-11-07T14:15:50,396 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T14:15:50,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:50,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:50,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:15:50,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/cd3c7bcf9bf1416cb49eabbe28422a22 is 50, key is test_row_0/A:col10/1730988950417/Put/seqid=0 2024-11-07T14:15:50,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989010439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742289_1465 (size=16681) 2024-11-07T14:15:50,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/cd3c7bcf9bf1416cb49eabbe28422a22 2024-11-07T14:15:50,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989010440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989010440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989010441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/a2b3bc3075344f868b09b079868110a4 is 50, key is test_row_0/B:col10/1730988950417/Put/seqid=0 2024-11-07T14:15:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989010442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742290_1466 (size=12001) 2024-11-07T14:15:50,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T14:15:50,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:50,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T14:15:50,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989010552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989010553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989010553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989010562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989010562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T14:15:50,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:50,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989010763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989010765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989010765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989010772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989010772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:50,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T14:15:50,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T14:15:50,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/a2b3bc3075344f868b09b079868110a4 2024-11-07T14:15:50,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/68593edc23384d1e8512e8cffc697caa is 50, key is test_row_0/C:col10/1730988950417/Put/seqid=0 2024-11-07T14:15:50,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742291_1467 (size=12001) 2024-11-07T14:15:50,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T14:15:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:50,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:51,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989011072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989011072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989011082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989011082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989011082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T14:15:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:51,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:51,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T14:15:51,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:51,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:51,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/68593edc23384d1e8512e8cffc697caa 2024-11-07T14:15:51,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/cd3c7bcf9bf1416cb49eabbe28422a22 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/cd3c7bcf9bf1416cb49eabbe28422a22 2024-11-07T14:15:51,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/cd3c7bcf9bf1416cb49eabbe28422a22, entries=250, sequenceid=39, filesize=16.3 K 2024-11-07T14:15:51,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/a2b3bc3075344f868b09b079868110a4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a2b3bc3075344f868b09b079868110a4 2024-11-07T14:15:51,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a2b3bc3075344f868b09b079868110a4, entries=150, sequenceid=39, filesize=11.7 K 2024-11-07T14:15:51,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/68593edc23384d1e8512e8cffc697caa as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/68593edc23384d1e8512e8cffc697caa 2024-11-07T14:15:51,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/68593edc23384d1e8512e8cffc697caa, entries=150, sequenceid=39, filesize=11.7 K 2024-11-07T14:15:51,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 758cb99e0ef7c32067829f2f2f90dc44 in 884ms, sequenceid=39, compaction requested=false 2024-11-07T14:15:51,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T14:15:51,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T14:15:51,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:51,427 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:15:51,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/b809486875c140079bb2bbf450b5302f is 50, key is test_row_0/A:col10/1730988950440/Put/seqid=0 2024-11-07T14:15:51,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742292_1468 (size=12001) 2024-11-07T14:15:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:51,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:51,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989011626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989011627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989011634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989011635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989011636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989011742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989011742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989011742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989011752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989011752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,838 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/b809486875c140079bb2bbf450b5302f 2024-11-07T14:15:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/e808f2e864ea4055aec383c1e23fe7d0 is 50, key is test_row_0/B:col10/1730988950440/Put/seqid=0 2024-11-07T14:15:51,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742293_1469 (size=12001) 2024-11-07T14:15:51,930 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/e808f2e864ea4055aec383c1e23fe7d0 2024-11-07T14:15:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989011952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989011952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989011952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989011962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989011963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:51,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/83af349e4f1c4b1b8e5c5c8ce5ee7203 is 50, key is test_row_0/C:col10/1730988950440/Put/seqid=0 2024-11-07T14:15:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742294_1470 (size=12001) 2024-11-07T14:15:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989012272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989012272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989012272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989012273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989012273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T14:15:52,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/83af349e4f1c4b1b8e5c5c8ce5ee7203 2024-11-07T14:15:52,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/b809486875c140079bb2bbf450b5302f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b809486875c140079bb2bbf450b5302f 2024-11-07T14:15:52,417 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b809486875c140079bb2bbf450b5302f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T14:15:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/e808f2e864ea4055aec383c1e23fe7d0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e808f2e864ea4055aec383c1e23fe7d0 2024-11-07T14:15:52,422 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e808f2e864ea4055aec383c1e23fe7d0, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T14:15:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/83af349e4f1c4b1b8e5c5c8ce5ee7203 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/83af349e4f1c4b1b8e5c5c8ce5ee7203 2024-11-07T14:15:52,426 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/83af349e4f1c4b1b8e5c5c8ce5ee7203, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T14:15:52,427 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 758cb99e0ef7c32067829f2f2f90dc44 in 1001ms, sequenceid=51, compaction requested=true 2024-11-07T14:15:52,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:52,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:52,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-07T14:15:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-07T14:15:52,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-07T14:15:52,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2330 sec 2024-11-07T14:15:52,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2410 sec 2024-11-07T14:15:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:52,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:52,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:15:52,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/fefc34a2c2124511aea8969324c48ab4 is 50, key is test_row_0/A:col10/1730988951635/Put/seqid=0 2024-11-07T14:15:52,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742295_1471 (size=14341) 2024-11-07T14:15:52,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/fefc34a2c2124511aea8969324c48ab4 2024-11-07T14:15:52,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989012801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989012803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989012811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/cac82e2989704a9b9276538989e6fcf8 is 50, key is test_row_0/B:col10/1730988951635/Put/seqid=0 2024-11-07T14:15:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742296_1472 (size=12001) 2024-11-07T14:15:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989012811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989012813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/cac82e2989704a9b9276538989e6fcf8 2024-11-07T14:15:52,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/a7ffec13725b4a049ad90676cd26df85 is 50, key is test_row_0/C:col10/1730988951635/Put/seqid=0 2024-11-07T14:15:52,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742297_1473 (size=12001) 2024-11-07T14:15:52,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/a7ffec13725b4a049ad90676cd26df85 2024-11-07T14:15:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/fefc34a2c2124511aea8969324c48ab4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fefc34a2c2124511aea8969324c48ab4 2024-11-07T14:15:52,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fefc34a2c2124511aea8969324c48ab4, entries=200, sequenceid=76, filesize=14.0 K 2024-11-07T14:15:52,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/cac82e2989704a9b9276538989e6fcf8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cac82e2989704a9b9276538989e6fcf8 2024-11-07T14:15:52,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cac82e2989704a9b9276538989e6fcf8, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T14:15:52,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/a7ffec13725b4a049ad90676cd26df85 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a7ffec13725b4a049ad90676cd26df85 2024-11-07T14:15:52,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a7ffec13725b4a049ad90676cd26df85, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T14:15:52,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 758cb99e0ef7c32067829f2f2f90dc44 in 80ms, sequenceid=76, compaction requested=true 2024-11-07T14:15:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:52,875 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:52,875 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:52,876 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:52,876 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:15:52,876 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:52,876 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/acdf09be141c40188b308f4b86be9aa1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a2b3bc3075344f868b09b079868110a4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e808f2e864ea4055aec383c1e23fe7d0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cac82e2989704a9b9276538989e6fcf8] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=46.9 K 2024-11-07T14:15:52,876 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:52,876 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:15:52,876 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:52,877 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1250fce944b440878e82a2131c8301a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/cd3c7bcf9bf1416cb49eabbe28422a22, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b809486875c140079bb2bbf450b5302f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fefc34a2c2124511aea8969324c48ab4] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=53.7 K 2024-11-07T14:15:52,877 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting acdf09be141c40188b308f4b86be9aa1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988950199 2024-11-07T14:15:52,877 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1250fce944b440878e82a2131c8301a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988950199 2024-11-07T14:15:52,877 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b3bc3075344f868b09b079868110a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1730988950265 2024-11-07T14:15:52,877 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd3c7bcf9bf1416cb49eabbe28422a22, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1730988950265 2024-11-07T14:15:52,878 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e808f2e864ea4055aec383c1e23fe7d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730988950439 2024-11-07T14:15:52,878 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting b809486875c140079bb2bbf450b5302f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730988950439 2024-11-07T14:15:52,878 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting cac82e2989704a9b9276538989e6fcf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988951633 2024-11-07T14:15:52,878 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting fefc34a2c2124511aea8969324c48ab4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988951633 2024-11-07T14:15:52,885 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:52,885 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:52,885 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/042efc8e884342698b868406a0a8bd31 is 50, key is test_row_0/A:col10/1730988951635/Put/seqid=0 2024-11-07T14:15:52,885 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/433322143a0440bb84260d650ba8c461 is 50, key is test_row_0/B:col10/1730988951635/Put/seqid=0 2024-11-07T14:15:52,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742298_1474 (size=12139) 2024-11-07T14:15:52,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742299_1475 (size=12139) 2024-11-07T14:15:52,897 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/433322143a0440bb84260d650ba8c461 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/433322143a0440bb84260d650ba8c461 2024-11-07T14:15:52,901 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into 433322143a0440bb84260d650ba8c461(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:52,901 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:52,901 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=12, startTime=1730988952875; duration=0sec 2024-11-07T14:15:52,901 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:52,901 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:15:52,901 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:52,902 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:52,902 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:15:52,902 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:52,902 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/4079f92164024697979d8579919f8830, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/68593edc23384d1e8512e8cffc697caa, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/83af349e4f1c4b1b8e5c5c8ce5ee7203, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a7ffec13725b4a049ad90676cd26df85] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=46.9 K 2024-11-07T14:15:52,902 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4079f92164024697979d8579919f8830, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730988950199 2024-11-07T14:15:52,902 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 68593edc23384d1e8512e8cffc697caa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1730988950265 2024-11-07T14:15:52,903 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 83af349e4f1c4b1b8e5c5c8ce5ee7203, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730988950439 2024-11-07T14:15:52,903 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting a7ffec13725b4a049ad90676cd26df85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988951633 2024-11-07T14:15:52,910 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:52,911 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/e703c12f36ff4716976b46c6ad80c788 is 50, key is test_row_0/C:col10/1730988951635/Put/seqid=0 2024-11-07T14:15:52,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742300_1476 (size=12139) 2024-11-07T14:15:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:52,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:15:52,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/ecf6da4d9dcf49ba85555914156a3510 is 50, key is test_row_0/A:col10/1730988952926/Put/seqid=0 2024-11-07T14:15:52,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742301_1477 (size=19021) 2024-11-07T14:15:52,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989012965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989012966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989012967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989012967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:52,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989012968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989013082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989013082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989013082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989013082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989013092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,129 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:15:53,296 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/042efc8e884342698b868406a0a8bd31 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/042efc8e884342698b868406a0a8bd31 2024-11-07T14:15:53,300 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into 042efc8e884342698b868406a0a8bd31(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:53,300 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:53,300 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=12, startTime=1730988952875; duration=0sec 2024-11-07T14:15:53,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,300 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989013292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,300 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:15:53,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989013292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989013292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989013292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989013302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,318 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/e703c12f36ff4716976b46c6ad80c788 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e703c12f36ff4716976b46c6ad80c788 2024-11-07T14:15:53,322 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into e703c12f36ff4716976b46c6ad80c788(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:53,322 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:53,322 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=12, startTime=1730988952875; duration=0sec 2024-11-07T14:15:53,322 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:53,322 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:15:53,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/ecf6da4d9dcf49ba85555914156a3510 2024-11-07T14:15:53,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/8728d202da6e45b5a50caaae5b0a9eb1 is 50, key is test_row_0/B:col10/1730988952926/Put/seqid=0 2024-11-07T14:15:53,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742302_1478 (size=12001) 2024-11-07T14:15:53,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989013601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989013612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989013612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989013612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989013612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:53,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/8728d202da6e45b5a50caaae5b0a9eb1 2024-11-07T14:15:53,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/fedd7c1333ed474b9289000091db404c is 50, key is test_row_0/C:col10/1730988952926/Put/seqid=0 2024-11-07T14:15:53,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742303_1479 (size=12001) 2024-11-07T14:15:54,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989014122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:54,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989014122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989014123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989014123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:54,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989014132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:54,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/fedd7c1333ed474b9289000091db404c 2024-11-07T14:15:54,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/ecf6da4d9dcf49ba85555914156a3510 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/ecf6da4d9dcf49ba85555914156a3510 2024-11-07T14:15:54,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/ecf6da4d9dcf49ba85555914156a3510, entries=300, sequenceid=91, filesize=18.6 K 2024-11-07T14:15:54,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/8728d202da6e45b5a50caaae5b0a9eb1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/8728d202da6e45b5a50caaae5b0a9eb1 2024-11-07T14:15:54,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/8728d202da6e45b5a50caaae5b0a9eb1, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T14:15:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/fedd7c1333ed474b9289000091db404c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fedd7c1333ed474b9289000091db404c 2024-11-07T14:15:54,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fedd7c1333ed474b9289000091db404c, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T14:15:54,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 758cb99e0ef7c32067829f2f2f90dc44 in 1241ms, sequenceid=91, compaction requested=false 2024-11-07T14:15:54,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T14:15:54,362 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:54,362 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-07T14:15:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T14:15:54,364 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:54,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:54,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T14:15:54,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T14:15:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:54,517 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T14:15:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/3fee9fa402704cd48772fd0ff809a05d is 50, key is test_row_0/A:col10/1730988952967/Put/seqid=0 2024-11-07T14:15:54,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742304_1480 (size=12001) 2024-11-07T14:15:54,541 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/3fee9fa402704cd48772fd0ff809a05d 2024-11-07T14:15:54,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/f38f34c7d83344d69ba0d67b03350f78 is 50, key is test_row_0/B:col10/1730988952967/Put/seqid=0 2024-11-07T14:15:54,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742305_1481 (size=12001) 2024-11-07T14:15:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T14:15:54,956 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/f38f34c7d83344d69ba0d67b03350f78 2024-11-07T14:15:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/f9569a635f144c37a63dbf55dce43ad4 is 50, key is test_row_0/C:col10/1730988952967/Put/seqid=0 2024-11-07T14:15:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T14:15:55,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742306_1482 (size=12001) 2024-11-07T14:15:55,005 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/f9569a635f144c37a63dbf55dce43ad4 2024-11-07T14:15:55,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/3fee9fa402704cd48772fd0ff809a05d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3fee9fa402704cd48772fd0ff809a05d 2024-11-07T14:15:55,017 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3fee9fa402704cd48772fd0ff809a05d, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T14:15:55,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/f38f34c7d83344d69ba0d67b03350f78 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f38f34c7d83344d69ba0d67b03350f78 2024-11-07T14:15:55,101 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f38f34c7d83344d69ba0d67b03350f78, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T14:15:55,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/f9569a635f144c37a63dbf55dce43ad4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/f9569a635f144c37a63dbf55dce43ad4 2024-11-07T14:15:55,115 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/f9569a635f144c37a63dbf55dce43ad4, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T14:15:55,116 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 758cb99e0ef7c32067829f2f2f90dc44 in 599ms, sequenceid=115, compaction requested=true 2024-11-07T14:15:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-07T14:15:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-07T14:15:55,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-07T14:15:55,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 753 msec 2024-11-07T14:15:55,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 759 msec 2024-11-07T14:15:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:55,156 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:55,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:15:55,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/121c88fa5abc401b95f2642b31481d6b is 50, key is test_row_0/A:col10/1730988955137/Put/seqid=0 2024-11-07T14:15:55,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742307_1483 (size=14341) 2024-11-07T14:15:55,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/121c88fa5abc401b95f2642b31481d6b 2024-11-07T14:15:55,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/2b9ccbc41c54478991d22c51c51e926f is 50, key is test_row_0/B:col10/1730988955137/Put/seqid=0 2024-11-07T14:15:55,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742308_1484 (size=12001) 2024-11-07T14:15:55,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989015200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989015209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989015214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989015216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989015219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989015322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989015332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989015343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989015343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989015343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T14:15:55,502 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:55,503 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-07T14:15:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T14:15:55,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:55,507 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:55,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989015534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989015542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989015563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989015563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989015563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/2b9ccbc41c54478991d22c51c51e926f 2024-11-07T14:15:55,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/2b19a4ba96074d99b83cb5ccb3a5c7ce is 50, key is test_row_0/C:col10/1730988955137/Put/seqid=0 2024-11-07T14:15:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T14:15:55,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742309_1485 (size=12001) 2024-11-07T14:15:55,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/2b19a4ba96074d99b83cb5ccb3a5c7ce 2024-11-07T14:15:55,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/121c88fa5abc401b95f2642b31481d6b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/121c88fa5abc401b95f2642b31481d6b 2024-11-07T14:15:55,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/121c88fa5abc401b95f2642b31481d6b, entries=200, sequenceid=126, filesize=14.0 K 2024-11-07T14:15:55,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/2b9ccbc41c54478991d22c51c51e926f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b9ccbc41c54478991d22c51c51e926f 2024-11-07T14:15:55,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b9ccbc41c54478991d22c51c51e926f, entries=150, sequenceid=126, filesize=11.7 K 2024-11-07T14:15:55,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/2b19a4ba96074d99b83cb5ccb3a5c7ce as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/2b19a4ba96074d99b83cb5ccb3a5c7ce 2024-11-07T14:15:55,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T14:15:55,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:55,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:55,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:55,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:55,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:55,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/2b19a4ba96074d99b83cb5ccb3a5c7ce, entries=150, sequenceid=126, filesize=11.7 K 2024-11-07T14:15:55,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 758cb99e0ef7c32067829f2f2f90dc44 in 510ms, sequenceid=126, compaction requested=true 2024-11-07T14:15:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:55,667 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:55,667 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:55,668 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57502 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:55,668 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:15:55,668 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:55,668 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/042efc8e884342698b868406a0a8bd31, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/ecf6da4d9dcf49ba85555914156a3510, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3fee9fa402704cd48772fd0ff809a05d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/121c88fa5abc401b95f2642b31481d6b] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=56.2 K 2024-11-07T14:15:55,668 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:55,668 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:15:55,668 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:55,668 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 042efc8e884342698b868406a0a8bd31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988951633 2024-11-07T14:15:55,668 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/433322143a0440bb84260d650ba8c461, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/8728d202da6e45b5a50caaae5b0a9eb1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f38f34c7d83344d69ba0d67b03350f78, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b9ccbc41c54478991d22c51c51e926f] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=47.0 K 2024-11-07T14:15:55,669 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf6da4d9dcf49ba85555914156a3510, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730988952796 2024-11-07T14:15:55,669 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 433322143a0440bb84260d650ba8c461, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988951633 2024-11-07T14:15:55,669 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fee9fa402704cd48772fd0ff809a05d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730988952964 2024-11-07T14:15:55,669 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8728d202da6e45b5a50caaae5b0a9eb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730988952796 2024-11-07T14:15:55,669 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 121c88fa5abc401b95f2642b31481d6b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730988955137 2024-11-07T14:15:55,669 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f38f34c7d83344d69ba0d67b03350f78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730988952964 2024-11-07T14:15:55,670 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9ccbc41c54478991d22c51c51e926f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730988955137 2024-11-07T14:15:55,679 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:55,679 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/8589fd14f03049c4885412ac6985450d is 50, key is test_row_0/A:col10/1730988955137/Put/seqid=0 2024-11-07T14:15:55,683 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#413 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:55,683 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/82bf139a434e4d0a8d5368feda2b01a6 is 50, key is test_row_0/B:col10/1730988955137/Put/seqid=0 2024-11-07T14:15:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742310_1486 (size=12275) 2024-11-07T14:15:55,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742311_1487 (size=12275) 2024-11-07T14:15:55,716 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/82bf139a434e4d0a8d5368feda2b01a6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/82bf139a434e4d0a8d5368feda2b01a6 2024-11-07T14:15:55,722 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into 82bf139a434e4d0a8d5368feda2b01a6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:55,722 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:55,722 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=12, startTime=1730988955667; duration=0sec 2024-11-07T14:15:55,722 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:55,722 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:15:55,722 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:15:55,723 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:15:55,723 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:15:55,723 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:55,723 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e703c12f36ff4716976b46c6ad80c788, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fedd7c1333ed474b9289000091db404c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/f9569a635f144c37a63dbf55dce43ad4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/2b19a4ba96074d99b83cb5ccb3a5c7ce] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=47.0 K 2024-11-07T14:15:55,723 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e703c12f36ff4716976b46c6ad80c788, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730988951633 2024-11-07T14:15:55,724 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting fedd7c1333ed474b9289000091db404c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730988952796 2024-11-07T14:15:55,724 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f9569a635f144c37a63dbf55dce43ad4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730988952964 2024-11-07T14:15:55,724 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b19a4ba96074d99b83cb5ccb3a5c7ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730988955137 2024-11-07T14:15:55,737 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:55,738 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/df28565a49f247e78ee567e5e13076b1 is 50, key is test_row_0/C:col10/1730988955137/Put/seqid=0 2024-11-07T14:15:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742312_1488 (size=12275) 2024-11-07T14:15:55,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T14:15:55,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:55,813 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:15:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/4110da3ca21b486aab61e1ca42566f6f is 50, key is test_row_0/A:col10/1730988955211/Put/seqid=0 2024-11-07T14:15:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T14:15:55,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742313_1489 (size=12151) 2024-11-07T14:15:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:55,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989015878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989015882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989015882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989015882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989015883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989015992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989015992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,108 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/8589fd14f03049c4885412ac6985450d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/8589fd14f03049c4885412ac6985450d 2024-11-07T14:15:56,111 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into 8589fd14f03049c4885412ac6985450d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:56,111 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:56,111 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=12, startTime=1730988955667; duration=0sec 2024-11-07T14:15:56,111 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:56,111 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:15:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T14:15:56,150 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/df28565a49f247e78ee567e5e13076b1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/df28565a49f247e78ee567e5e13076b1 2024-11-07T14:15:56,154 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into df28565a49f247e78ee567e5e13076b1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:56,154 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:56,154 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=12, startTime=1730988955667; duration=0sec 2024-11-07T14:15:56,154 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:56,154 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:15:56,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989016202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989016212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,229 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/4110da3ca21b486aab61e1ca42566f6f 2024-11-07T14:15:56,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/988e5436fd254d21b30c068dc5cff3d6 is 50, key is test_row_0/B:col10/1730988955211/Put/seqid=0 2024-11-07T14:15:56,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742314_1490 (size=12151) 2024-11-07T14:15:56,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989016392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:56,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989016392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989016402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989016514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989016522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:56,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:56,640 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/988e5436fd254d21b30c068dc5cff3d6 2024-11-07T14:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T14:15:56,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/5ce7d0ce14894bb5b5fcb115577833e0 is 50, key is test_row_0/C:col10/1730988955211/Put/seqid=0 2024-11-07T14:15:56,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742315_1491 (size=12151) 2024-11-07T14:15:56,660 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/5ce7d0ce14894bb5b5fcb115577833e0 2024-11-07T14:15:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/4110da3ca21b486aab61e1ca42566f6f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/4110da3ca21b486aab61e1ca42566f6f 2024-11-07T14:15:56,682 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/4110da3ca21b486aab61e1ca42566f6f, entries=150, sequenceid=152, filesize=11.9 K 2024-11-07T14:15:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/988e5436fd254d21b30c068dc5cff3d6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/988e5436fd254d21b30c068dc5cff3d6 2024-11-07T14:15:56,695 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/988e5436fd254d21b30c068dc5cff3d6, entries=150, sequenceid=152, filesize=11.9 K 2024-11-07T14:15:56,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/5ce7d0ce14894bb5b5fcb115577833e0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/5ce7d0ce14894bb5b5fcb115577833e0 2024-11-07T14:15:56,705 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/5ce7d0ce14894bb5b5fcb115577833e0, entries=150, sequenceid=152, filesize=11.9 K 2024-11-07T14:15:56,706 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 758cb99e0ef7c32067829f2f2f90dc44 in 893ms, sequenceid=152, compaction requested=false 2024-11-07T14:15:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-07T14:15:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-07T14:15:56,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-07T14:15:56,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2010 sec 2024-11-07T14:15:56,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.2080 sec 2024-11-07T14:15:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:57,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:57,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T14:15:57,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/51bfe53914714813b491f370263fb55c is 50, key is test_row_0/A:col10/1730988957025/Put/seqid=0 2024-11-07T14:15:57,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742316_1492 (size=14541) 2024-11-07T14:15:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989017131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989017145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989017252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989017262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989017402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,410 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 758cb99e0ef7c32067829f2f2f90dc44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:15:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989017412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,415 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_2' in 758cb99e0ef7c32067829f2f2f90dc44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:15:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989017422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,431 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 758cb99e0ef7c32067829f2f2f90dc44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:15:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/51bfe53914714813b491f370263fb55c 2024-11-07T14:15:57,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/b752a08c853844caac91013c1892deaf is 50, key is test_row_0/B:col10/1730988957025/Put/seqid=0 2024-11-07T14:15:57,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989017472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989017472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742317_1493 (size=12151) 2024-11-07T14:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T14:15:57,654 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:57,654 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-07T14:15:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T14:15:57,657 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:57,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:57,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T14:15:57,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989017792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989017793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:57,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T14:15:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:57,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:57,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:57,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/b752a08c853844caac91013c1892deaf 2024-11-07T14:15:57,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/cd0e25112f3644d6bbdf39abdf0a1494 is 50, key is test_row_0/C:col10/1730988957025/Put/seqid=0 2024-11-07T14:15:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742318_1494 (size=12151) 2024-11-07T14:15:57,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T14:15:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:57,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T14:15:58,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T14:15:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:58,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T14:15:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T14:15:58,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/cd0e25112f3644d6bbdf39abdf0a1494 2024-11-07T14:15:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/51bfe53914714813b491f370263fb55c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/51bfe53914714813b491f370263fb55c 2024-11-07T14:15:58,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/51bfe53914714813b491f370263fb55c, entries=200, sequenceid=166, filesize=14.2 K 2024-11-07T14:15:58,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/b752a08c853844caac91013c1892deaf as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b752a08c853844caac91013c1892deaf 2024-11-07T14:15:58,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989018312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:58,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989018312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:58,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b752a08c853844caac91013c1892deaf, entries=150, sequenceid=166, filesize=11.9 K 2024-11-07T14:15:58,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/cd0e25112f3644d6bbdf39abdf0a1494 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cd0e25112f3644d6bbdf39abdf0a1494 2024-11-07T14:15:58,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cd0e25112f3644d6bbdf39abdf0a1494, entries=150, sequenceid=166, filesize=11.9 K 2024-11-07T14:15:58,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 758cb99e0ef7c32067829f2f2f90dc44 in 1304ms, sequenceid=166, compaction requested=true 2024-11-07T14:15:58,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:58,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:58,329 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:58,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:58,330 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:58,330 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:58,330 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:15:58,330 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,330 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/8589fd14f03049c4885412ac6985450d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/4110da3ca21b486aab61e1ca42566f6f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/51bfe53914714813b491f370263fb55c] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=38.1 K 2024-11-07T14:15:58,331 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:58,331 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:15:58,331 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,331 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/82bf139a434e4d0a8d5368feda2b01a6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/988e5436fd254d21b30c068dc5cff3d6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b752a08c853844caac91013c1892deaf] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=35.7 K 2024-11-07T14:15:58,331 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8589fd14f03049c4885412ac6985450d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730988955137 2024-11-07T14:15:58,331 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 82bf139a434e4d0a8d5368feda2b01a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730988955137 2024-11-07T14:15:58,331 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 988e5436fd254d21b30c068dc5cff3d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730988955198 2024-11-07T14:15:58,331 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4110da3ca21b486aab61e1ca42566f6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730988955198 2024-11-07T14:15:58,331 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b752a08c853844caac91013c1892deaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988955872 2024-11-07T14:15:58,331 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51bfe53914714813b491f370263fb55c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988955872 2024-11-07T14:15:58,337 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:58,338 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/f42581c6b44a4a549ae0d7fb345800e2 is 50, key is test_row_0/B:col10/1730988957025/Put/seqid=0 2024-11-07T14:15:58,340 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:58,340 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/43560e01d9334b0588c9188371a016b6 is 50, key is test_row_0/A:col10/1730988957025/Put/seqid=0 2024-11-07T14:15:58,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742319_1495 (size=12527) 2024-11-07T14:15:58,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742320_1496 (size=12527) 2024-11-07T14:15:58,350 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/f42581c6b44a4a549ae0d7fb345800e2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f42581c6b44a4a549ae0d7fb345800e2 2024-11-07T14:15:58,354 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into f42581c6b44a4a549ae0d7fb345800e2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:58,354 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:58,354 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=13, startTime=1730988958330; duration=0sec 2024-11-07T14:15:58,354 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:15:58,354 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:15:58,354 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:58,355 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:15:58,355 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:15:58,355 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,355 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/df28565a49f247e78ee567e5e13076b1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/5ce7d0ce14894bb5b5fcb115577833e0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cd0e25112f3644d6bbdf39abdf0a1494] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=35.7 K 2024-11-07T14:15:58,356 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting df28565a49f247e78ee567e5e13076b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730988955137 2024-11-07T14:15:58,356 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ce7d0ce14894bb5b5fcb115577833e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730988955198 2024-11-07T14:15:58,356 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting cd0e25112f3644d6bbdf39abdf0a1494, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988955872 2024-11-07T14:15:58,364 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:15:58,364 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/dfddb67b669d4d1d9355681e6621afdd is 50, key is test_row_0/C:col10/1730988957025/Put/seqid=0 2024-11-07T14:15:58,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742321_1497 (size=12527) 2024-11-07T14:15:58,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T14:15:58,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,429 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:15:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/db63701d89dc45dfb2057b7d082e0ae8 is 50, key is test_row_0/A:col10/1730988957124/Put/seqid=0 2024-11-07T14:15:58,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742322_1498 (size=12151) 2024-11-07T14:15:58,439 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/db63701d89dc45dfb2057b7d082e0ae8 2024-11-07T14:15:58,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/b7aa6235ef0b41a8b1176ac644ab9b1d is 50, key is test_row_0/B:col10/1730988957124/Put/seqid=0 2024-11-07T14:15:58,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742323_1499 (size=12151) 2024-11-07T14:15:58,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/b7aa6235ef0b41a8b1176ac644ab9b1d 2024-11-07T14:15:58,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/20f3b35c8ad447dabeaf59238bf5f099 is 50, key is test_row_0/C:col10/1730988957124/Put/seqid=0 2024-11-07T14:15:58,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742324_1500 (size=12151) 2024-11-07T14:15:58,754 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/43560e01d9334b0588c9188371a016b6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/43560e01d9334b0588c9188371a016b6 2024-11-07T14:15:58,758 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into 43560e01d9334b0588c9188371a016b6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:58,758 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:58,758 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=13, startTime=1730988958329; duration=0sec 2024-11-07T14:15:58,758 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:58,758 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:15:58,780 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/dfddb67b669d4d1d9355681e6621afdd as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dfddb67b669d4d1d9355681e6621afdd 2024-11-07T14:15:58,787 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into dfddb67b669d4d1d9355681e6621afdd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:15:58,787 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:58,787 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=13, startTime=1730988958330; duration=0sec 2024-11-07T14:15:58,787 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:58,787 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:15:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T14:15:58,865 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/20f3b35c8ad447dabeaf59238bf5f099 2024-11-07T14:15:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/db63701d89dc45dfb2057b7d082e0ae8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/db63701d89dc45dfb2057b7d082e0ae8 2024-11-07T14:15:58,873 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/db63701d89dc45dfb2057b7d082e0ae8, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T14:15:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/b7aa6235ef0b41a8b1176ac644ab9b1d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b7aa6235ef0b41a8b1176ac644ab9b1d 2024-11-07T14:15:58,894 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b7aa6235ef0b41a8b1176ac644ab9b1d, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T14:15:58,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/20f3b35c8ad447dabeaf59238bf5f099 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/20f3b35c8ad447dabeaf59238bf5f099 2024-11-07T14:15:58,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/20f3b35c8ad447dabeaf59238bf5f099, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T14:15:58,900 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 758cb99e0ef7c32067829f2f2f90dc44 in 471ms, sequenceid=191, compaction requested=false 2024-11-07T14:15:58,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:58,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:58,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-07T14:15:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-07T14:15:58,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-07T14:15:58,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2420 sec 2024-11-07T14:15:58,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.2500 sec 2024-11-07T14:15:59,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:15:59,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:15:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:15:59,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/5e56f4e00d1d4bb6acc7885b9dfbd62d is 50, key is test_row_0/A:col10/1730988959361/Put/seqid=0 2024-11-07T14:15:59,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742325_1501 (size=14541) 2024-11-07T14:15:59,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/5e56f4e00d1d4bb6acc7885b9dfbd62d 2024-11-07T14:15:59,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/3a44b85367cf41ebafe727bbe3457bf0 is 50, key is test_row_0/B:col10/1730988959361/Put/seqid=0 2024-11-07T14:15:59,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989019462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989019463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989019467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742326_1502 (size=12151) 2024-11-07T14:15:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989019479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989019478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989019583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989019586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989019594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989019603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989019603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989019792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T14:15:59,803 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:15:59,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:15:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989019803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-07T14:15:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T14:15:59,815 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:15:59,819 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:15:59,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:15:59,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989019813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989019822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:15:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989019832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:15:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:15:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:15:59,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/3a44b85367cf41ebafe727bbe3457bf0 2024-11-07T14:15:59,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/42fb2fbdd9784956bf0a4e9053adc351 is 50, key is test_row_0/C:col10/1730988959361/Put/seqid=0 2024-11-07T14:15:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T14:15:59,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742327_1503 (size=12151) 2024-11-07T14:15:59,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/42fb2fbdd9784956bf0a4e9053adc351 2024-11-07T14:15:59,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/5e56f4e00d1d4bb6acc7885b9dfbd62d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/5e56f4e00d1d4bb6acc7885b9dfbd62d 2024-11-07T14:15:59,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T14:15:59,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/5e56f4e00d1d4bb6acc7885b9dfbd62d, entries=200, sequenceid=204, filesize=14.2 K 2024-11-07T14:15:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:15:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:15:59,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:59,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/3a44b85367cf41ebafe727bbe3457bf0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/3a44b85367cf41ebafe727bbe3457bf0 2024-11-07T14:15:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:15:59,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/3a44b85367cf41ebafe727bbe3457bf0, entries=150, sequenceid=204, filesize=11.9 K 2024-11-07T14:15:59,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/42fb2fbdd9784956bf0a4e9053adc351 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/42fb2fbdd9784956bf0a4e9053adc351 2024-11-07T14:15:59,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/42fb2fbdd9784956bf0a4e9053adc351, entries=150, sequenceid=204, filesize=11.9 K 2024-11-07T14:15:59,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 758cb99e0ef7c32067829f2f2f90dc44 in 625ms, sequenceid=204, compaction requested=true 2024-11-07T14:15:59,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:15:59,990 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:59,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:15:59,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:59,990 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:15:59,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:15:59,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:15:59,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:15:59,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:00,007 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:00,007 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:16:00,007 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:00,007 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/43560e01d9334b0588c9188371a016b6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/db63701d89dc45dfb2057b7d082e0ae8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/5e56f4e00d1d4bb6acc7885b9dfbd62d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=38.3 K 2024-11-07T14:16:00,010 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:00,010 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:16:00,010 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:00,010 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f42581c6b44a4a549ae0d7fb345800e2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b7aa6235ef0b41a8b1176ac644ab9b1d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/3a44b85367cf41ebafe727bbe3457bf0] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.0 K 2024-11-07T14:16:00,010 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43560e01d9334b0588c9188371a016b6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988955872 2024-11-07T14:16:00,014 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f42581c6b44a4a549ae0d7fb345800e2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988955872 2024-11-07T14:16:00,014 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting db63701d89dc45dfb2057b7d082e0ae8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730988957115 2024-11-07T14:16:00,017 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e56f4e00d1d4bb6acc7885b9dfbd62d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730988959344 2024-11-07T14:16:00,017 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b7aa6235ef0b41a8b1176ac644ab9b1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730988957115 2024-11-07T14:16:00,019 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a44b85367cf41ebafe727bbe3457bf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730988959344 2024-11-07T14:16:00,029 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:00,029 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/041471a7ff33445b8289a8599f239860 is 50, key is test_row_0/A:col10/1730988959361/Put/seqid=0 2024-11-07T14:16:00,045 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:00,045 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/fa72e73614ed46a3ab32f166a1da659e is 50, key is test_row_0/B:col10/1730988959361/Put/seqid=0 2024-11-07T14:16:00,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742328_1504 (size=12629) 2024-11-07T14:16:00,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742329_1505 (size=12629) 2024-11-07T14:16:00,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T14:16:00,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:00,130 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T14:16:00,131 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/041471a7ff33445b8289a8599f239860 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/041471a7ff33445b8289a8599f239860 2024-11-07T14:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:00,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T14:16:00,145 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/fa72e73614ed46a3ab32f166a1da659e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/fa72e73614ed46a3ab32f166a1da659e 2024-11-07T14:16:00,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/772ed65beb244f2bafe4563684c10700 is 50, key is test_row_0/A:col10/1730988960129/Put/seqid=0 2024-11-07T14:16:00,159 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into 041471a7ff33445b8289a8599f239860(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:00,159 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:00,159 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=13, startTime=1730988959990; duration=0sec 2024-11-07T14:16:00,159 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:00,159 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:16:00,160 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:00,168 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into fa72e73614ed46a3ab32f166a1da659e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:00,168 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:00,169 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=13, startTime=1730988959990; duration=0sec 2024-11-07T14:16:00,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:00,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:16:00,173 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:00,173 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:16:00,173 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:00,173 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dfddb67b669d4d1d9355681e6621afdd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/20f3b35c8ad447dabeaf59238bf5f099, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/42fb2fbdd9784956bf0a4e9053adc351] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.0 K 2024-11-07T14:16:00,175 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfddb67b669d4d1d9355681e6621afdd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730988955872 2024-11-07T14:16:00,176 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20f3b35c8ad447dabeaf59238bf5f099, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730988957115 2024-11-07T14:16:00,177 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42fb2fbdd9784956bf0a4e9053adc351, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730988959344 2024-11-07T14:16:00,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989020161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742330_1506 (size=16931) 2024-11-07T14:16:00,196 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/772ed65beb244f2bafe4563684c10700 2024-11-07T14:16:00,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989020168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989020168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989020170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989020172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,201 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:00,201 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/cb185434cd0a469fbee66115c39769e4 is 50, key is test_row_0/C:col10/1730988959361/Put/seqid=0 2024-11-07T14:16:00,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/dce5ecfdab3e4f399473080d1a69c0e0 is 50, key is test_row_0/B:col10/1730988960129/Put/seqid=0 2024-11-07T14:16:00,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742331_1507 (size=12629) 2024-11-07T14:16:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742332_1508 (size=12151) 2024-11-07T14:16:00,307 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/dce5ecfdab3e4f399473080d1a69c0e0 2024-11-07T14:16:00,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989020303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989020303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989020303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989020303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989020303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/73c7595d6dc24fc6b91a77280ec2c396 is 50, key is test_row_0/C:col10/1730988960129/Put/seqid=0 2024-11-07T14:16:00,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742333_1509 (size=12151) 2024-11-07T14:16:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T14:16:00,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989020523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989020523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989020524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989020526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989020524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,675 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/cb185434cd0a469fbee66115c39769e4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cb185434cd0a469fbee66115c39769e4 2024-11-07T14:16:00,686 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into cb185434cd0a469fbee66115c39769e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:00,686 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:00,686 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=13, startTime=1730988959990; duration=0sec 2024-11-07T14:16:00,686 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:00,686 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:16:00,810 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/73c7595d6dc24fc6b91a77280ec2c396 2024-11-07T14:16:00,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/772ed65beb244f2bafe4563684c10700 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/772ed65beb244f2bafe4563684c10700 2024-11-07T14:16:00,827 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/772ed65beb244f2bafe4563684c10700, entries=250, sequenceid=232, filesize=16.5 K 2024-11-07T14:16:00,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/dce5ecfdab3e4f399473080d1a69c0e0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/dce5ecfdab3e4f399473080d1a69c0e0 2024-11-07T14:16:00,834 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/dce5ecfdab3e4f399473080d1a69c0e0, entries=150, sequenceid=232, filesize=11.9 K 2024-11-07T14:16:00,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/73c7595d6dc24fc6b91a77280ec2c396 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/73c7595d6dc24fc6b91a77280ec2c396 2024-11-07T14:16:00,844 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/73c7595d6dc24fc6b91a77280ec2c396, entries=150, sequenceid=232, filesize=11.9 K 2024-11-07T14:16:00,845 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 758cb99e0ef7c32067829f2f2f90dc44 in 715ms, sequenceid=232, compaction requested=false 2024-11-07T14:16:00,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:00,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:00,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-07T14:16:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-07T14:16:00,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-07T14:16:00,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2024-11-07T14:16:00,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0460 sec 2024-11-07T14:16:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:00,855 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:00,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:16:00,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/2e9a54e4e2364083a20305099b7d1173 is 50, key is test_row_0/A:col10/1730988960853/Put/seqid=0 2024-11-07T14:16:00,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742334_1510 (size=12151) 2024-11-07T14:16:00,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/2e9a54e4e2364083a20305099b7d1173 2024-11-07T14:16:00,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d7124e3ebd2e47c98d627b4a3aceb89d is 50, key is test_row_0/B:col10/1730988960853/Put/seqid=0 2024-11-07T14:16:00,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742335_1511 (size=12151) 2024-11-07T14:16:00,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d7124e3ebd2e47c98d627b4a3aceb89d 2024-11-07T14:16:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989020922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989020932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989020935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989020938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989020939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:00,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/d386ae0a536548488284b88d98046919 is 50, key is test_row_0/C:col10/1730988960853/Put/seqid=0 2024-11-07T14:16:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T14:16:00,958 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:00,963 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-07T14:16:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T14:16:00,972 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:00,981 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:00,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:01,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742336_1512 (size=12151) 2024-11-07T14:16:01,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989021043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T14:16:01,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989021062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989021062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989021063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989021063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-07T14:16:01,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:01,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:01,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:01,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:01,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:01,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989021263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T14:16:01,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989021283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989021286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989021286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989021292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-07T14:16:01,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:01,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:01,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:01,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:01,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/d386ae0a536548488284b88d98046919 2024-11-07T14:16:01,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/2e9a54e4e2364083a20305099b7d1173 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2e9a54e4e2364083a20305099b7d1173 2024-11-07T14:16:01,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2e9a54e4e2364083a20305099b7d1173, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T14:16:01,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d7124e3ebd2e47c98d627b4a3aceb89d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d7124e3ebd2e47c98d627b4a3aceb89d 2024-11-07T14:16:01,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d7124e3ebd2e47c98d627b4a3aceb89d, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T14:16:01,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/d386ae0a536548488284b88d98046919 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/d386ae0a536548488284b88d98046919 2024-11-07T14:16:01,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/d386ae0a536548488284b88d98046919, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T14:16:01,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 758cb99e0ef7c32067829f2f2f90dc44 in 603ms, sequenceid=248, compaction requested=true 2024-11-07T14:16:01,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:01,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:01,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:01,458 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:01,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:01,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:01,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:01,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:16:01,459 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:01,459 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:01,460 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:16:01,460 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:01,460 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/041471a7ff33445b8289a8599f239860, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/772ed65beb244f2bafe4563684c10700, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2e9a54e4e2364083a20305099b7d1173] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=40.7 K 2024-11-07T14:16:01,460 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:01,460 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:16:01,460 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:01,460 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/fa72e73614ed46a3ab32f166a1da659e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/dce5ecfdab3e4f399473080d1a69c0e0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d7124e3ebd2e47c98d627b4a3aceb89d] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.1 K 2024-11-07T14:16:01,460 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 041471a7ff33445b8289a8599f239860, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730988959344 2024-11-07T14:16:01,461 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting fa72e73614ed46a3ab32f166a1da659e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730988959344 2024-11-07T14:16:01,461 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 772ed65beb244f2bafe4563684c10700, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1730988959458 2024-11-07T14:16:01,461 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting dce5ecfdab3e4f399473080d1a69c0e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1730988959458 2024-11-07T14:16:01,461 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e9a54e4e2364083a20305099b7d1173, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988960153 2024-11-07T14:16:01,461 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d7124e3ebd2e47c98d627b4a3aceb89d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988960153 2024-11-07T14:16:01,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-07T14:16:01,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:01,462 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T14:16:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/b9b4b59a3f4b497a94d1c120992407f8 is 50, key is test_row_0/A:col10/1730988960934/Put/seqid=0 2024-11-07T14:16:01,474 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:01,475 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/6065d09d41384b08a1c96e848b8559f8 is 50, key is test_row_0/B:col10/1730988960853/Put/seqid=0 2024-11-07T14:16:01,477 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:01,477 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/41a3542414a949b096fd5c85ad006e76 is 50, key is test_row_0/A:col10/1730988960853/Put/seqid=0 2024-11-07T14:16:01,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742337_1513 (size=12301) 2024-11-07T14:16:01,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742339_1515 (size=12731) 2024-11-07T14:16:01,554 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/41a3542414a949b096fd5c85ad006e76 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/41a3542414a949b096fd5c85ad006e76 2024-11-07T14:16:01,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742338_1514 (size=12731) 2024-11-07T14:16:01,566 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into 41a3542414a949b096fd5c85ad006e76(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:01,566 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:01,566 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=13, startTime=1730988961458; duration=0sec 2024-11-07T14:16:01,566 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:01,566 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:16:01,566 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:01,569 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/6065d09d41384b08a1c96e848b8559f8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/6065d09d41384b08a1c96e848b8559f8 2024-11-07T14:16:01,569 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:01,569 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:16:01,569 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:01,570 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cb185434cd0a469fbee66115c39769e4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/73c7595d6dc24fc6b91a77280ec2c396, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/d386ae0a536548488284b88d98046919] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.1 K 2024-11-07T14:16:01,573 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb185434cd0a469fbee66115c39769e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730988959344 2024-11-07T14:16:01,573 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c7595d6dc24fc6b91a77280ec2c396, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1730988959458 2024-11-07T14:16:01,576 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting d386ae0a536548488284b88d98046919, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988960153 2024-11-07T14:16:01,585 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into 6065d09d41384b08a1c96e848b8559f8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:01,585 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:01,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:01,585 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=13, startTime=1730988961458; duration=0sec 2024-11-07T14:16:01,586 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:01,586 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:16:01,591 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:01,591 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/41c61bf516d9459f89503b49304abdf8 is 50, key is test_row_0/C:col10/1730988960853/Put/seqid=0 2024-11-07T14:16:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T14:16:01,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742340_1516 (size=12731) 2024-11-07T14:16:01,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989021628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989021637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989021648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989021648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989021650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989021752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989021762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989021762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989021762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989021762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,931 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/b9b4b59a3f4b497a94d1c120992407f8 2024-11-07T14:16:01,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/73af2ee080c743279e75302411131fbb is 50, key is test_row_0/B:col10/1730988960934/Put/seqid=0 2024-11-07T14:16:01,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742341_1517 (size=12301) 2024-11-07T14:16:01,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989021962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989021972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989021975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989021983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989021983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,021 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/41c61bf516d9459f89503b49304abdf8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/41c61bf516d9459f89503b49304abdf8 2024-11-07T14:16:02,026 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into 41c61bf516d9459f89503b49304abdf8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:02,026 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:02,026 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=13, startTime=1730988961458; duration=0sec 2024-11-07T14:16:02,026 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:02,026 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:16:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T14:16:02,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989022271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989022292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989022292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989022292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989022292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,360 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/73af2ee080c743279e75302411131fbb 2024-11-07T14:16:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/3ec1e01abac6475f9dd08492b92b6ec1 is 50, key is test_row_0/C:col10/1730988960934/Put/seqid=0 2024-11-07T14:16:02,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742342_1518 (size=12301) 2024-11-07T14:16:02,771 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/3ec1e01abac6475f9dd08492b92b6ec1 2024-11-07T14:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/b9b4b59a3f4b497a94d1c120992407f8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b9b4b59a3f4b497a94d1c120992407f8 2024-11-07T14:16:02,777 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b9b4b59a3f4b497a94d1c120992407f8, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T14:16:02,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/73af2ee080c743279e75302411131fbb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/73af2ee080c743279e75302411131fbb 2024-11-07T14:16:02,783 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/73af2ee080c743279e75302411131fbb, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T14:16:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/3ec1e01abac6475f9dd08492b92b6ec1 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3ec1e01abac6475f9dd08492b92b6ec1 2024-11-07T14:16:02,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3ec1e01abac6475f9dd08492b92b6ec1, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T14:16:02,787 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 758cb99e0ef7c32067829f2f2f90dc44 in 1325ms, sequenceid=271, compaction requested=false 2024-11-07T14:16:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-07T14:16:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-07T14:16:02,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-07T14:16:02,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8070 sec 2024-11-07T14:16:02,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.8270 sec 2024-11-07T14:16:02,799 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:02,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T14:16:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:02,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/e12dade043214eb8afe93f07d9053fc0 is 50, key is test_row_0/A:col10/1730988962797/Put/seqid=0 2024-11-07T14:16:02,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742343_1519 (size=12301) 2024-11-07T14:16:02,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989022840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989022839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989022872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989022873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989022873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989022982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989022982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989022991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989022991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989022992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T14:16:03,113 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:03,113 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-07T14:16:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T14:16:03,115 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:03,116 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:03,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:03,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989023192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989023192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989023202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989023202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989023202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/e12dade043214eb8afe93f07d9053fc0 2024-11-07T14:16:03,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/cef1c84383894a4a9dcc6280efd1b4c3 is 50, key is test_row_0/B:col10/1730988962797/Put/seqid=0 2024-11-07T14:16:03,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742344_1520 (size=12301) 2024-11-07T14:16:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T14:16:03,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T14:16:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T14:16:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T14:16:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989023512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989023512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989023512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989023512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989023513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:03,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T14:16:03,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:03,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:03,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/cef1c84383894a4a9dcc6280efd1b4c3 2024-11-07T14:16:03,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6dbdcb190412476280369b607d7d28dc is 50, key is test_row_0/C:col10/1730988962797/Put/seqid=0 2024-11-07T14:16:03,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742345_1521 (size=12301) 2024-11-07T14:16:03,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6dbdcb190412476280369b607d7d28dc 2024-11-07T14:16:03,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/e12dade043214eb8afe93f07d9053fc0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/e12dade043214eb8afe93f07d9053fc0 2024-11-07T14:16:03,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/e12dade043214eb8afe93f07d9053fc0, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T14:16:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/cef1c84383894a4a9dcc6280efd1b4c3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cef1c84383894a4a9dcc6280efd1b4c3 2024-11-07T14:16:03,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cef1c84383894a4a9dcc6280efd1b4c3, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T14:16:03,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6dbdcb190412476280369b607d7d28dc as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6dbdcb190412476280369b607d7d28dc 2024-11-07T14:16:03,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6dbdcb190412476280369b607d7d28dc, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T14:16:03,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 758cb99e0ef7c32067829f2f2f90dc44 in 842ms, sequenceid=288, compaction requested=true 2024-11-07T14:16:03,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:03,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:03,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:03,641 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:03,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:03,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:03,641 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:03,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:03,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:03,642 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:03,642 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:03,642 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:16:03,642 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:16:03,642 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,642 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,642 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/41a3542414a949b096fd5c85ad006e76, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b9b4b59a3f4b497a94d1c120992407f8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/e12dade043214eb8afe93f07d9053fc0] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.5 K 2024-11-07T14:16:03,642 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/6065d09d41384b08a1c96e848b8559f8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/73af2ee080c743279e75302411131fbb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cef1c84383894a4a9dcc6280efd1b4c3] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.5 K 2024-11-07T14:16:03,643 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6065d09d41384b08a1c96e848b8559f8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988960153 2024-11-07T14:16:03,643 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a3542414a949b096fd5c85ad006e76, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988960153 2024-11-07T14:16:03,643 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9b4b59a3f4b497a94d1c120992407f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730988960906 2024-11-07T14:16:03,643 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 73af2ee080c743279e75302411131fbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730988960906 2024-11-07T14:16:03,643 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e12dade043214eb8afe93f07d9053fc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988961618 2024-11-07T14:16:03,643 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting cef1c84383894a4a9dcc6280efd1b4c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988961618 2024-11-07T14:16:03,650 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:03,650 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:03,650 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/f4ca6fbb5ef24aee946bedd1bf5ca6d5 is 50, key is test_row_0/A:col10/1730988962797/Put/seqid=0 2024-11-07T14:16:03,650 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/95c8983b15664cb4a82cb8740da18119 is 50, key is test_row_0/B:col10/1730988962797/Put/seqid=0 2024-11-07T14:16:03,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742347_1523 (size=12983) 2024-11-07T14:16:03,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742346_1522 (size=12983) 2024-11-07T14:16:03,659 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/f4ca6fbb5ef24aee946bedd1bf5ca6d5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/f4ca6fbb5ef24aee946bedd1bf5ca6d5 2024-11-07T14:16:03,663 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into f4ca6fbb5ef24aee946bedd1bf5ca6d5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:03,663 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:03,663 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=13, startTime=1730988963641; duration=0sec 2024-11-07T14:16:03,663 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:03,663 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:16:03,663 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:03,664 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:03,664 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:16:03,664 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,664 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/41c61bf516d9459f89503b49304abdf8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3ec1e01abac6475f9dd08492b92b6ec1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6dbdcb190412476280369b607d7d28dc] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.5 K 2024-11-07T14:16:03,665 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c61bf516d9459f89503b49304abdf8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730988960153 2024-11-07T14:16:03,666 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ec1e01abac6475f9dd08492b92b6ec1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730988960906 2024-11-07T14:16:03,666 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dbdcb190412476280369b607d7d28dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988961618 2024-11-07T14:16:03,672 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:03,672 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/e51ba4f48771443794a6def80ce0b36e is 50, key is test_row_0/C:col10/1730988962797/Put/seqid=0 2024-11-07T14:16:03,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742348_1524 (size=12983) 2024-11-07T14:16:03,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T14:16:03,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:03,727 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T14:16:03,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/1cef0eae35fc4f7ba6c62e168d4d82fa is 50, key is test_row_0/A:col10/1730988962838/Put/seqid=0 2024-11-07T14:16:03,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742349_1525 (size=12301) 2024-11-07T14:16:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T14:16:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:04,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:04,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989024037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989024038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989024040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989024046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989024047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,058 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/95c8983b15664cb4a82cb8740da18119 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/95c8983b15664cb4a82cb8740da18119 2024-11-07T14:16:04,064 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into 95c8983b15664cb4a82cb8740da18119(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:04,064 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:04,064 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=13, startTime=1730988963641; duration=0sec 2024-11-07T14:16:04,064 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:04,064 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:16:04,082 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/e51ba4f48771443794a6def80ce0b36e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e51ba4f48771443794a6def80ce0b36e 2024-11-07T14:16:04,086 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into e51ba4f48771443794a6def80ce0b36e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:04,086 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:04,086 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=13, startTime=1730988963641; duration=0sec 2024-11-07T14:16:04,086 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:04,086 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:16:04,135 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/1cef0eae35fc4f7ba6c62e168d4d82fa 2024-11-07T14:16:04,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/bc66bb311b174b72bff5dd3d7afa7104 is 50, key is test_row_0/B:col10/1730988962838/Put/seqid=0 2024-11-07T14:16:04,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742350_1526 (size=12301) 2024-11-07T14:16:04,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989024152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989024152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989024162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989024162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989024162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T14:16:04,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989024372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989024372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989024372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989024372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989024382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,550 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/bc66bb311b174b72bff5dd3d7afa7104 2024-11-07T14:16:04,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/21024b9a38764ab3a89fc0e09531f195 is 50, key is test_row_0/C:col10/1730988962838/Put/seqid=0 2024-11-07T14:16:04,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742351_1527 (size=12301) 2024-11-07T14:16:04,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989024692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989024692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989024692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989024692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989024695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:04,985 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/21024b9a38764ab3a89fc0e09531f195 2024-11-07T14:16:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/1cef0eae35fc4f7ba6c62e168d4d82fa as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1cef0eae35fc4f7ba6c62e168d4d82fa 2024-11-07T14:16:04,998 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1cef0eae35fc4f7ba6c62e168d4d82fa, entries=150, sequenceid=312, filesize=12.0 K 2024-11-07T14:16:04,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/bc66bb311b174b72bff5dd3d7afa7104 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/bc66bb311b174b72bff5dd3d7afa7104 2024-11-07T14:16:05,003 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/bc66bb311b174b72bff5dd3d7afa7104, entries=150, sequenceid=312, filesize=12.0 K 2024-11-07T14:16:05,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/21024b9a38764ab3a89fc0e09531f195 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/21024b9a38764ab3a89fc0e09531f195 2024-11-07T14:16:05,008 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/21024b9a38764ab3a89fc0e09531f195, entries=150, sequenceid=312, filesize=12.0 K 2024-11-07T14:16:05,009 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 758cb99e0ef7c32067829f2f2f90dc44 in 1281ms, sequenceid=312, compaction requested=false 2024-11-07T14:16:05,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:05,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-07T14:16:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-07T14:16:05,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-07T14:16:05,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8950 sec 2024-11-07T14:16:05,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.9010 sec 2024-11-07T14:16:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:05,208 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:05,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T14:16:05,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/28a26d40f31e411fbbe55f2afe9ec9d4 is 50, key is test_row_0/A:col10/1730988965206/Put/seqid=0 2024-11-07T14:16:05,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742352_1528 (size=14741) 2024-11-07T14:16:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T14:16:05,262 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:05,263 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-07T14:16:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T14:16:05,267 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:05,268 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:05,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:05,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989025289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989025289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989025304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989025303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989025305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T14:16:05,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989025412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989025412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T14:16:05,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989025422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989025422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989025422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T14:16:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T14:16:05,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989025624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989025624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/28a26d40f31e411fbbe55f2afe9ec9d4 2024-11-07T14:16:05,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d3878913a39b411c950fdfc991c30569 is 50, key is test_row_0/B:col10/1730988965206/Put/seqid=0 2024-11-07T14:16:05,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742353_1529 (size=12301) 2024-11-07T14:16:05,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989025642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989025642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989025643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T14:16:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T14:16:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:05,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T14:16:05,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989025932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989025932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989025963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989025963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989025963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:05,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:06,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T14:16:06,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:06,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d3878913a39b411c950fdfc991c30569 2024-11-07T14:16:06,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/0bc2b568c9de402287e7fe5f6d1d8037 is 50, key is test_row_0/C:col10/1730988965206/Put/seqid=0 2024-11-07T14:16:06,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742354_1530 (size=12301) 2024-11-07T14:16:06,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T14:16:06,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:06,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T14:16:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:06,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T14:16:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989026452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989026452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:06,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/0bc2b568c9de402287e7fe5f6d1d8037 2024-11-07T14:16:06,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989026472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:06,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989026472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:06,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/28a26d40f31e411fbbe55f2afe9ec9d4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/28a26d40f31e411fbbe55f2afe9ec9d4 2024-11-07T14:16:06,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/28a26d40f31e411fbbe55f2afe9ec9d4, entries=200, sequenceid=328, filesize=14.4 K 2024-11-07T14:16:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d3878913a39b411c950fdfc991c30569 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d3878913a39b411c950fdfc991c30569 2024-11-07T14:16:06,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d3878913a39b411c950fdfc991c30569, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T14:16:06,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/0bc2b568c9de402287e7fe5f6d1d8037 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/0bc2b568c9de402287e7fe5f6d1d8037 2024-11-07T14:16:06,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:06,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/0bc2b568c9de402287e7fe5f6d1d8037, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T14:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989026482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:06,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 758cb99e0ef7c32067829f2f2f90dc44 in 1286ms, sequenceid=328, compaction requested=true 2024-11-07T14:16:06,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:06,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:06,494 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:06,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:06,495 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:06,495 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:06,495 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:06,495 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:16:06,495 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:16:06,495 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,495 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,495 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/f4ca6fbb5ef24aee946bedd1bf5ca6d5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1cef0eae35fc4f7ba6c62e168d4d82fa, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/28a26d40f31e411fbbe55f2afe9ec9d4] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=39.1 K 2024-11-07T14:16:06,495 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/95c8983b15664cb4a82cb8740da18119, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/bc66bb311b174b72bff5dd3d7afa7104, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d3878913a39b411c950fdfc991c30569] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.7 K 2024-11-07T14:16:06,496 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4ca6fbb5ef24aee946bedd1bf5ca6d5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988961618 2024-11-07T14:16:06,496 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 95c8983b15664cb4a82cb8740da18119, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988961618 2024-11-07T14:16:06,496 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cef0eae35fc4f7ba6c62e168d4d82fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730988962838 2024-11-07T14:16:06,496 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting bc66bb311b174b72bff5dd3d7afa7104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730988962838 2024-11-07T14:16:06,496 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a26d40f31e411fbbe55f2afe9ec9d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988964032 2024-11-07T14:16:06,496 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d3878913a39b411c950fdfc991c30569, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988964032 2024-11-07T14:16:06,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T14:16:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,499 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T14:16:06,513 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:06,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/638a7c9732994fa98615a910ff797088 is 50, key is test_row_0/A:col10/1730988965286/Put/seqid=0 2024-11-07T14:16:06,514 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/4dd16e069941482c823a6671bb92af58 is 50, key is test_row_0/B:col10/1730988965206/Put/seqid=0 2024-11-07T14:16:06,515 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:06,515 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/a9138a439dd14b0bafcec80c4fb2a56a is 50, key is test_row_0/A:col10/1730988965206/Put/seqid=0 2024-11-07T14:16:06,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742355_1531 (size=13085) 2024-11-07T14:16:06,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742356_1532 (size=12301) 2024-11-07T14:16:06,525 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/638a7c9732994fa98615a910ff797088 2024-11-07T14:16:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d0d91adc76694df49b2b70e9b80bcdc5 is 50, key is test_row_0/B:col10/1730988965286/Put/seqid=0 2024-11-07T14:16:06,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742357_1533 (size=13085) 2024-11-07T14:16:06,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742358_1534 (size=12301) 2024-11-07T14:16:06,926 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/a9138a439dd14b0bafcec80c4fb2a56a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/a9138a439dd14b0bafcec80c4fb2a56a 2024-11-07T14:16:06,934 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into a9138a439dd14b0bafcec80c4fb2a56a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:06,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:06,934 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=13, startTime=1730988966494; duration=0sec 2024-11-07T14:16:06,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:06,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:16:06,934 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:06,935 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:06,935 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:16:06,935 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:06,935 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e51ba4f48771443794a6def80ce0b36e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/21024b9a38764ab3a89fc0e09531f195, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/0bc2b568c9de402287e7fe5f6d1d8037] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.7 K 2024-11-07T14:16:06,936 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e51ba4f48771443794a6def80ce0b36e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730988961618 2024-11-07T14:16:06,936 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21024b9a38764ab3a89fc0e09531f195, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1730988962838 2024-11-07T14:16:06,936 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc2b568c9de402287e7fe5f6d1d8037, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988964032 2024-11-07T14:16:06,947 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/4dd16e069941482c823a6671bb92af58 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/4dd16e069941482c823a6671bb92af58 2024-11-07T14:16:06,950 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d0d91adc76694df49b2b70e9b80bcdc5 2024-11-07T14:16:06,953 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:06,954 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into 4dd16e069941482c823a6671bb92af58(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:06,954 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:06,954 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=13, startTime=1730988966494; duration=0sec 2024-11-07T14:16:06,955 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:06,955 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:16:06,955 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/3b2437fb262f4007beba1ec555f8ff46 is 50, key is test_row_0/C:col10/1730988965206/Put/seqid=0 2024-11-07T14:16:06,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742359_1535 (size=13085) 2024-11-07T14:16:06,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/e2b7bb4d7c3b4ef1928347a0a734b089 is 50, key is test_row_0/C:col10/1730988965286/Put/seqid=0 2024-11-07T14:16:06,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742360_1536 (size=12301) 2024-11-07T14:16:06,978 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/e2b7bb4d7c3b4ef1928347a0a734b089 2024-11-07T14:16:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/638a7c9732994fa98615a910ff797088 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/638a7c9732994fa98615a910ff797088 2024-11-07T14:16:06,990 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/638a7c9732994fa98615a910ff797088, entries=150, sequenceid=350, filesize=12.0 K 2024-11-07T14:16:06,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d0d91adc76694df49b2b70e9b80bcdc5 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0d91adc76694df49b2b70e9b80bcdc5 2024-11-07T14:16:07,000 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0d91adc76694df49b2b70e9b80bcdc5, entries=150, sequenceid=350, filesize=12.0 K 2024-11-07T14:16:07,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/e2b7bb4d7c3b4ef1928347a0a734b089 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e2b7bb4d7c3b4ef1928347a0a734b089 2024-11-07T14:16:07,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e2b7bb4d7c3b4ef1928347a0a734b089, entries=150, sequenceid=350, filesize=12.0 K 2024-11-07T14:16:07,007 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 758cb99e0ef7c32067829f2f2f90dc44 in 508ms, sequenceid=350, compaction requested=false 2024-11-07T14:16:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-07T14:16:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-07T14:16:07,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-07T14:16:07,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-11-07T14:16:07,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.7480 sec 2024-11-07T14:16:07,373 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/3b2437fb262f4007beba1ec555f8ff46 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3b2437fb262f4007beba1ec555f8ff46 2024-11-07T14:16:07,381 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into 3b2437fb262f4007beba1ec555f8ff46(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:07,381 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:07,381 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=13, startTime=1730988966495; duration=0sec 2024-11-07T14:16:07,382 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:07,382 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:16:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T14:16:07,412 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:07,412 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-07T14:16:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T14:16:07,414 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:07,415 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:07,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:07,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:07,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:16:07,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/0a02085408eb46f5be9229c1df15df03 is 50, key is test_row_0/A:col10/1730988967468/Put/seqid=0 2024-11-07T14:16:07,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742361_1537 (size=14741) 2024-11-07T14:16:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T14:16:07,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989027544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989027547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989027548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989027554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989027555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T14:16:07,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:07,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:07,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:07,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989027662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989027662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989027662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989027672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989027672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T14:16:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:07,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T14:16:07,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T14:16:07,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:07,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:07,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989027873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989027873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989027873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/0a02085408eb46f5be9229c1df15df03 2024-11-07T14:16:07,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989027882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989027882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:07,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/e9250eda711a44b1a05db30b33c8fc05 is 50, key is test_row_0/B:col10/1730988967468/Put/seqid=0 2024-11-07T14:16:07,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742362_1538 (size=12301) 2024-11-07T14:16:08,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T14:16:08,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:08,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T14:16:08,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T14:16:08,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989028182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989028183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989028183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989028202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989028202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/e9250eda711a44b1a05db30b33c8fc05 2024-11-07T14:16:08,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc is 50, key is test_row_0/C:col10/1730988967468/Put/seqid=0 2024-11-07T14:16:08,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T14:16:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:08,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742363_1539 (size=12301) 2024-11-07T14:16:08,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc 2024-11-07T14:16:08,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/0a02085408eb46f5be9229c1df15df03 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/0a02085408eb46f5be9229c1df15df03 2024-11-07T14:16:08,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/0a02085408eb46f5be9229c1df15df03, entries=200, sequenceid=364, filesize=14.4 K 2024-11-07T14:16:08,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/e9250eda711a44b1a05db30b33c8fc05 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e9250eda711a44b1a05db30b33c8fc05 2024-11-07T14:16:08,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e9250eda711a44b1a05db30b33c8fc05, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T14:16:08,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc 2024-11-07T14:16:08,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T14:16:08,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 758cb99e0ef7c32067829f2f2f90dc44 in 906ms, sequenceid=364, compaction requested=true 2024-11-07T14:16:08,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:08,385 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:08,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:08,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:08,385 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:08,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:08,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:08,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:08,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:08,386 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:08,386 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:16:08,386 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:08,386 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:16:08,386 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,387 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,387 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/a9138a439dd14b0bafcec80c4fb2a56a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/638a7c9732994fa98615a910ff797088, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/0a02085408eb46f5be9229c1df15df03] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=39.2 K 2024-11-07T14:16:08,387 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/4dd16e069941482c823a6671bb92af58, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0d91adc76694df49b2b70e9b80bcdc5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e9250eda711a44b1a05db30b33c8fc05] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.8 K 2024-11-07T14:16:08,387 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd16e069941482c823a6671bb92af58, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988964032 2024-11-07T14:16:08,387 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9138a439dd14b0bafcec80c4fb2a56a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988964032 2024-11-07T14:16:08,387 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638a7c9732994fa98615a910ff797088, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1730988965286 2024-11-07T14:16:08,387 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting d0d91adc76694df49b2b70e9b80bcdc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1730988965286 2024-11-07T14:16:08,388 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a02085408eb46f5be9229c1df15df03, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730988967465 2024-11-07T14:16:08,388 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e9250eda711a44b1a05db30b33c8fc05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730988967468 2024-11-07T14:16:08,398 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:08,399 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/2f93bb3bc7494a9d9c9f07c8f40fae0c is 50, key is test_row_0/A:col10/1730988967468/Put/seqid=0 2024-11-07T14:16:08,400 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:08,401 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/9b978e21bf24401d869384f249f123de is 50, key is test_row_0/B:col10/1730988967468/Put/seqid=0 2024-11-07T14:16:08,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742364_1540 (size=13187) 2024-11-07T14:16:08,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742365_1541 (size=13187) 2024-11-07T14:16:08,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-07T14:16:08,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,488 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:16:08,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/83d082608d744ebea712d4a6770758b4 is 50, key is test_row_0/A:col10/1730988967554/Put/seqid=0 2024-11-07T14:16:08,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742366_1542 (size=12301) 2024-11-07T14:16:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T14:16:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:08,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:08,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989028722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989028722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989028724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989028730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989028731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,820 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/2f93bb3bc7494a9d9c9f07c8f40fae0c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2f93bb3bc7494a9d9c9f07c8f40fae0c 2024-11-07T14:16:08,834 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/9b978e21bf24401d869384f249f123de as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/9b978e21bf24401d869384f249f123de 2024-11-07T14:16:08,835 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into 2f93bb3bc7494a9d9c9f07c8f40fae0c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:08,835 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:08,835 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=13, startTime=1730988968385; duration=0sec 2024-11-07T14:16:08,835 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:08,835 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:16:08,835 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:08,838 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:08,838 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:16:08,838 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,839 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3b2437fb262f4007beba1ec555f8ff46, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e2b7bb4d7c3b4ef1928347a0a734b089, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.8 K 2024-11-07T14:16:08,839 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b2437fb262f4007beba1ec555f8ff46, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730988964032 2024-11-07T14:16:08,839 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2b7bb4d7c3b4ef1928347a0a734b089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1730988965286 2024-11-07T14:16:08,840 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b53c4aa9b5440eaa3aa5e7eb56ffddc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730988967468 2024-11-07T14:16:08,841 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into 9b978e21bf24401d869384f249f123de(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:08,841 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:08,841 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=13, startTime=1730988968385; duration=0sec 2024-11-07T14:16:08,841 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:08,841 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:16:08,846 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:08,847 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/fe45dba2a65d4fe5b61e204f37ffd48e is 50, key is test_row_0/C:col10/1730988967468/Put/seqid=0 2024-11-07T14:16:08,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989028843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989028843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989028843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:08,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742367_1543 (size=13187) 2024-11-07T14:16:08,897 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/83d082608d744ebea712d4a6770758b4 2024-11-07T14:16:08,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/312a42a6d4424bf19b0be507f8453534 is 50, key is test_row_0/B:col10/1730988967554/Put/seqid=0 2024-11-07T14:16:08,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742368_1544 (size=12301) 2024-11-07T14:16:08,917 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/312a42a6d4424bf19b0be507f8453534 2024-11-07T14:16:08,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/79d07f4a226342a48114361a6d57d3b6 is 50, key is test_row_0/C:col10/1730988967554/Put/seqid=0 2024-11-07T14:16:08,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742369_1545 (size=12301) 2024-11-07T14:16:08,937 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/79d07f4a226342a48114361a6d57d3b6 2024-11-07T14:16:08,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/83d082608d744ebea712d4a6770758b4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/83d082608d744ebea712d4a6770758b4 2024-11-07T14:16:08,948 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/83d082608d744ebea712d4a6770758b4, entries=150, sequenceid=389, filesize=12.0 K 2024-11-07T14:16:08,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/312a42a6d4424bf19b0be507f8453534 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/312a42a6d4424bf19b0be507f8453534 2024-11-07T14:16:08,953 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/312a42a6d4424bf19b0be507f8453534, entries=150, sequenceid=389, filesize=12.0 K 2024-11-07T14:16:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/79d07f4a226342a48114361a6d57d3b6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/79d07f4a226342a48114361a6d57d3b6 2024-11-07T14:16:08,958 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/79d07f4a226342a48114361a6d57d3b6, entries=150, sequenceid=389, filesize=12.0 K 2024-11-07T14:16:08,960 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 758cb99e0ef7c32067829f2f2f90dc44 in 472ms, sequenceid=389, compaction requested=false 2024-11-07T14:16:08,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:08,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:08,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-07T14:16:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-07T14:16:08,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-07T14:16:08,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5460 sec 2024-11-07T14:16:08,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.5520 sec 2024-11-07T14:16:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:09,058 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:09,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:16:09,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/138f8eb5a3634b5f996211edbb8206d9 is 50, key is test_row_0/A:col10/1730988968729/Put/seqid=0 2024-11-07T14:16:09,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742370_1546 (size=14741) 2024-11-07T14:16:09,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/138f8eb5a3634b5f996211edbb8206d9 2024-11-07T14:16:09,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/0d45403db52e4b1e8843cfa1466fc344 is 50, key is test_row_0/B:col10/1730988968729/Put/seqid=0 2024-11-07T14:16:09,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742371_1547 (size=12301) 2024-11-07T14:16:09,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/0d45403db52e4b1e8843cfa1466fc344 2024-11-07T14:16:09,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/dc0871e7dc1f42ef85f0a190432f8a02 is 50, key is test_row_0/C:col10/1730988968729/Put/seqid=0 2024-11-07T14:16:09,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742372_1548 (size=12301) 2024-11-07T14:16:09,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/dc0871e7dc1f42ef85f0a190432f8a02 2024-11-07T14:16:09,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/138f8eb5a3634b5f996211edbb8206d9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/138f8eb5a3634b5f996211edbb8206d9 2024-11-07T14:16:09,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/138f8eb5a3634b5f996211edbb8206d9, entries=200, sequenceid=404, filesize=14.4 K 2024-11-07T14:16:09,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/0d45403db52e4b1e8843cfa1466fc344 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/0d45403db52e4b1e8843cfa1466fc344 2024-11-07T14:16:09,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/0d45403db52e4b1e8843cfa1466fc344, entries=150, sequenceid=404, filesize=12.0 K 2024-11-07T14:16:09,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/dc0871e7dc1f42ef85f0a190432f8a02 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dc0871e7dc1f42ef85f0a190432f8a02 2024-11-07T14:16:09,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dc0871e7dc1f42ef85f0a190432f8a02, entries=150, sequenceid=404, filesize=12.0 K 2024-11-07T14:16:09,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 758cb99e0ef7c32067829f2f2f90dc44 in 86ms, sequenceid=404, compaction requested=true 2024-11-07T14:16:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:09,145 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:16:09,146 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:09,146 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/A is initiating minor compaction (all files) 2024-11-07T14:16:09,146 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/A in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:09,146 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2f93bb3bc7494a9d9c9f07c8f40fae0c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/83d082608d744ebea712d4a6770758b4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/138f8eb5a3634b5f996211edbb8206d9] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=39.3 K 2024-11-07T14:16:09,146 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f93bb3bc7494a9d9c9f07c8f40fae0c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730988967468 2024-11-07T14:16:09,147 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d082608d744ebea712d4a6770758b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1730988967534 2024-11-07T14:16:09,147 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 138f8eb5a3634b5f996211edbb8206d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1730988968710 2024-11-07T14:16:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:09,151 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:09,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T14:16:09,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/104321e31d704c77b39bc2ab8e9aa2e6 is 50, key is test_row_0/A:col10/1730988969152/Put/seqid=0 2024-11-07T14:16:09,169 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#A#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:09,169 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/c2b637471ec448ad913bc8d661d9eaf6 is 50, key is test_row_0/A:col10/1730988968729/Put/seqid=0 2024-11-07T14:16:09,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742373_1549 (size=17181) 2024-11-07T14:16:09,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/104321e31d704c77b39bc2ab8e9aa2e6 2024-11-07T14:16:09,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989029172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989029178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/a004f1c44f4f46fab24b752e7d944ec2 is 50, key is test_row_0/B:col10/1730988969152/Put/seqid=0 2024-11-07T14:16:09,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742374_1550 (size=13289) 2024-11-07T14:16:09,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989029181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,201 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/c2b637471ec448ad913bc8d661d9eaf6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/c2b637471ec448ad913bc8d661d9eaf6 2024-11-07T14:16:09,207 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/A of 758cb99e0ef7c32067829f2f2f90dc44 into c2b637471ec448ad913bc8d661d9eaf6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:09,207 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:09,207 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/A, priority=13, startTime=1730988969145; duration=0sec 2024-11-07T14:16:09,207 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:16:09,207 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:16:09,207 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-07T14:16:09,208 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T14:16:09,208 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T14:16:09,208 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. because compaction request was cancelled 2024-11-07T14:16:09,209 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:16:09,209 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:09,209 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:09,210 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/B is initiating minor compaction (all files) 2024-11-07T14:16:09,210 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/B in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:09,210 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/9b978e21bf24401d869384f249f123de, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/312a42a6d4424bf19b0be507f8453534, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/0d45403db52e4b1e8843cfa1466fc344] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=36.9 K 2024-11-07T14:16:09,210 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b978e21bf24401d869384f249f123de, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730988967468 2024-11-07T14:16:09,210 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 312a42a6d4424bf19b0be507f8453534, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1730988967534 2024-11-07T14:16:09,211 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d45403db52e4b1e8843cfa1466fc344, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1730988968723 2024-11-07T14:16:09,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742375_1551 (size=12301) 2024-11-07T14:16:09,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/a004f1c44f4f46fab24b752e7d944ec2 2024-11-07T14:16:09,220 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#B#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:09,220 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/2b424f5f998341b6ada804398e8b0c52 is 50, key is test_row_0/B:col10/1730988968729/Put/seqid=0 2024-11-07T14:16:09,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/a1ac36710a7a417189103c075a11c9fb is 50, key is test_row_0/C:col10/1730988969152/Put/seqid=0 2024-11-07T14:16:09,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742376_1552 (size=13289) 2024-11-07T14:16:09,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742377_1553 (size=12301) 2024-11-07T14:16:09,261 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/fe45dba2a65d4fe5b61e204f37ffd48e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fe45dba2a65d4fe5b61e204f37ffd48e 2024-11-07T14:16:09,265 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into fe45dba2a65d4fe5b61e204f37ffd48e(size=12.9 K), total size for store is 36.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:09,265 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:09,265 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=13, startTime=1730988968386; duration=0sec 2024-11-07T14:16:09,265 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:09,265 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:16:09,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989029293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989029294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989029302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989029512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989029512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989029513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T14:16:09,562 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:09,562 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-07T14:16:09,564 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:09,565 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:09,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T14:16:09,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/a1ac36710a7a417189103c075a11c9fb 2024-11-07T14:16:09,640 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/2b424f5f998341b6ada804398e8b0c52 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b424f5f998341b6ada804398e8b0c52 2024-11-07T14:16:09,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/104321e31d704c77b39bc2ab8e9aa2e6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/104321e31d704c77b39bc2ab8e9aa2e6 2024-11-07T14:16:09,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/104321e31d704c77b39bc2ab8e9aa2e6, entries=250, sequenceid=430, filesize=16.8 K 2024-11-07T14:16:09,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/a004f1c44f4f46fab24b752e7d944ec2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a004f1c44f4f46fab24b752e7d944ec2 2024-11-07T14:16:09,648 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/B of 758cb99e0ef7c32067829f2f2f90dc44 into 2b424f5f998341b6ada804398e8b0c52(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:09,648 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:09,648 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/B, priority=13, startTime=1730988969145; duration=0sec 2024-11-07T14:16:09,648 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:09,648 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:16:09,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a004f1c44f4f46fab24b752e7d944ec2, entries=150, sequenceid=430, filesize=12.0 K 2024-11-07T14:16:09,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/a1ac36710a7a417189103c075a11c9fb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a1ac36710a7a417189103c075a11c9fb 2024-11-07T14:16:09,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a1ac36710a7a417189103c075a11c9fb, entries=150, sequenceid=430, filesize=12.0 K 2024-11-07T14:16:09,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 758cb99e0ef7c32067829f2f2f90dc44 in 506ms, sequenceid=430, compaction requested=true 2024-11-07T14:16:09,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:09,658 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T14:16:09,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:09,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:09,659 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T14:16:09,659 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T14:16:09,659 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T14:16:09,659 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. because compaction request was cancelled 2024-11-07T14:16:09,659 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:16:09,660 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T14:16:09,660 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T14:16:09,660 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. because compaction request was cancelled 2024-11-07T14:16:09,660 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:16:09,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:09,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:09,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:09,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:09,660 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:16:09,663 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:16:09,663 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 758cb99e0ef7c32067829f2f2f90dc44/C is initiating minor compaction (all files) 2024-11-07T14:16:09,663 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 758cb99e0ef7c32067829f2f2f90dc44/C in TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:09,663 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fe45dba2a65d4fe5b61e204f37ffd48e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/79d07f4a226342a48114361a6d57d3b6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dc0871e7dc1f42ef85f0a190432f8a02, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a1ac36710a7a417189103c075a11c9fb] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp, totalSize=48.9 K 2024-11-07T14:16:09,664 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe45dba2a65d4fe5b61e204f37ffd48e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730988967468 2024-11-07T14:16:09,664 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79d07f4a226342a48114361a6d57d3b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1730988967534 2024-11-07T14:16:09,664 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc0871e7dc1f42ef85f0a190432f8a02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1730988968723 2024-11-07T14:16:09,665 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ac36710a7a417189103c075a11c9fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1730988969108 2024-11-07T14:16:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T14:16:09,697 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 758cb99e0ef7c32067829f2f2f90dc44#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:09,698 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/cc08873bc88c4784b994a57e4d00f933 is 50, key is test_row_0/C:col10/1730988969152/Put/seqid=0 2024-11-07T14:16:09,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742378_1554 (size=13323) 2024-11-07T14:16:09,721 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/cc08873bc88c4784b994a57e4d00f933 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cc08873bc88c4784b994a57e4d00f933 2024-11-07T14:16:09,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-07T14:16:09,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:09,723 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T14:16:09,728 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 758cb99e0ef7c32067829f2f2f90dc44/C of 758cb99e0ef7c32067829f2f2f90dc44 into cc08873bc88c4784b994a57e4d00f933(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:09,728 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:09,728 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., storeName=758cb99e0ef7c32067829f2f2f90dc44/C, priority=12, startTime=1730988969660; duration=0sec 2024-11-07T14:16:09,728 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:09,728 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:16:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/591124bf762148f38ea0656925e7ef66 is 50, key is test_row_1/A:col10/1730988969178/Put/seqid=0 2024-11-07T14:16:09,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742379_1555 (size=9857) 2024-11-07T14:16:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:09,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. as already flushing 2024-11-07T14:16:09,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989029828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989029829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989029830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989029831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989029840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T14:16:09,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989029954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989029954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989029954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989029954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989029955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,149 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/591124bf762148f38ea0656925e7ef66 2024-11-07T14:16:10,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989030162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d0b2a4af83db48d08330e42ea62fd1d6 is 50, key is test_row_1/B:col10/1730988969178/Put/seqid=0 2024-11-07T14:16:10,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989030172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989030172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989030173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989030175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T14:16:10,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742380_1556 (size=9857) 2024-11-07T14:16:10,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d0b2a4af83db48d08330e42ea62fd1d6 2024-11-07T14:16:10,217 INFO [Thread-2080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2080. 2024-11-07T14:16:10,218 DEBUG [Thread-2080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:10,218 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,218 INFO [Thread-2078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2078. 2024-11-07T14:16:10,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/b65f1b3c48a248f6ac32756f743ad167 is 50, key is test_row_1/C:col10/1730988969178/Put/seqid=0 2024-11-07T14:16:10,218 DEBUG [Thread-2078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:10,218 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,218 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,218 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,219 INFO [Thread-2076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2076. 2024-11-07T14:16:10,220 DEBUG [Thread-2076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:10,220 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,220 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:10,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:10,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:10,228 INFO [Thread-2082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2082. 2024-11-07T14:16:10,228 DEBUG [Thread-2082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:10,228 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,228 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:10,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742381_1557 (size=9857) 2024-11-07T14:16:10,230 INFO [Thread-2074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2074. 2024-11-07T14:16:10,231 DEBUG [Thread-2074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:10,231 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,231 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:10,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51538 deadline: 1730989030482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51528 deadline: 1730989030482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51576 deadline: 1730989030493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51582 deadline: 1730989030493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51534 deadline: 1730989030493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 , the old value is region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., hostname=b7cc9d764de6,37737,1730988826985, seqNum=2 because the exception is null or not the one we care about 2024-11-07T14:16:10,630 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/b65f1b3c48a248f6ac32756f743ad167 2024-11-07T14:16:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/591124bf762148f38ea0656925e7ef66 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/591124bf762148f38ea0656925e7ef66 2024-11-07T14:16:10,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/591124bf762148f38ea0656925e7ef66, entries=100, sequenceid=443, filesize=9.6 K 2024-11-07T14:16:10,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/d0b2a4af83db48d08330e42ea62fd1d6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0b2a4af83db48d08330e42ea62fd1d6 2024-11-07T14:16:10,642 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0b2a4af83db48d08330e42ea62fd1d6, entries=100, sequenceid=443, filesize=9.6 K 2024-11-07T14:16:10,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/b65f1b3c48a248f6ac32756f743ad167 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/b65f1b3c48a248f6ac32756f743ad167 2024-11-07T14:16:10,645 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/b65f1b3c48a248f6ac32756f743ad167, entries=100, sequenceid=443, filesize=9.6 K 2024-11-07T14:16:10,646 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 758cb99e0ef7c32067829f2f2f90dc44 in 924ms, sequenceid=443, compaction requested=true 2024-11-07T14:16:10,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:10,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:10,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-07T14:16:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-07T14:16:10,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-07T14:16:10,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0830 sec 2024-11-07T14:16:10,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.0880 sec 2024-11-07T14:16:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T14:16:10,702 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:10,994 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:10,994 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-07T14:16:10,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T14:16:10,994 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-11-07T14:16:10,994 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:10,994 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:10,994 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,994 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,994 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,994 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:10,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:10,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:10,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/3819bb3119da496fb6401c2fe91deb68 is 50, key is test_row_0/A:col10/1730988969803/Put/seqid=0 2024-11-07T14:16:11,003 INFO [Thread-2067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2067. 2024-11-07T14:16:11,003 INFO [Thread-2071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2071. 2024-11-07T14:16:11,003 INFO [Thread-2065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2065. 2024-11-07T14:16:11,004 DEBUG [Thread-2067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:11,004 DEBUG [Thread-2065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:11,004 DEBUG [Thread-2071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:11,004 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:11,004 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:11,004 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:11,004 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:11,004 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:11,004 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:11,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:11,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:11,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1411 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4220 rows 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1406 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4217 rows 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1400 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4200 rows 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1404 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4212 rows 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1398 2024-11-07T14:16:11,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4194 rows 2024-11-07T14:16:11,005 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T14:16:11,005 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:16:11,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:11,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:11,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:11,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T14:16:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742382_1558 (size=12301) 2024-11-07T14:16:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T14:16:11,009 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988971009"}]},"ts":"1730988971009"} 2024-11-07T14:16:11,010 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T14:16:11,010 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T14:16:11,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:16:11,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, UNASSIGN}] 2024-11-07T14:16:11,013 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, UNASSIGN 2024-11-07T14:16:11,014 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=758cb99e0ef7c32067829f2f2f90dc44, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:11,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, UNASSIGN because future has completed 2024-11-07T14:16:11,015 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:16:11,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:16:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T14:16:11,168 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:11,168 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:16:11,168 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 758cb99e0ef7c32067829f2f2f90dc44, disabling compactions & flushes 2024-11-07T14:16:11,168 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T14:16:11,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/3819bb3119da496fb6401c2fe91deb68 2024-11-07T14:16:11,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/557cc868b3fe498385d9832a2c30a205 is 50, key is test_row_0/B:col10/1730988969803/Put/seqid=0 2024-11-07T14:16:11,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742383_1559 (size=12301) 2024-11-07T14:16:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T14:16:11,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/557cc868b3fe498385d9832a2c30a205 2024-11-07T14:16:11,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/14b7f2c68d0443aa92cb5b7ca043921d is 50, key is test_row_0/C:col10/1730988969803/Put/seqid=0 2024-11-07T14:16:11,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742384_1560 (size=12301) 2024-11-07T14:16:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T14:16:12,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/14b7f2c68d0443aa92cb5b7ca043921d 2024-11-07T14:16:12,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/3819bb3119da496fb6401c2fe91deb68 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3819bb3119da496fb6401c2fe91deb68 2024-11-07T14:16:12,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3819bb3119da496fb6401c2fe91deb68, entries=150, sequenceid=472, filesize=12.0 K 2024-11-07T14:16:12,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/557cc868b3fe498385d9832a2c30a205 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/557cc868b3fe498385d9832a2c30a205 2024-11-07T14:16:12,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/557cc868b3fe498385d9832a2c30a205, entries=150, sequenceid=472, filesize=12.0 K 2024-11-07T14:16:12,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/14b7f2c68d0443aa92cb5b7ca043921d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/14b7f2c68d0443aa92cb5b7ca043921d 2024-11-07T14:16:12,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/14b7f2c68d0443aa92cb5b7ca043921d, entries=150, sequenceid=472, filesize=12.0 K 2024-11-07T14:16:12,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=20.13 KB/20610 for 758cb99e0ef7c32067829f2f2f90dc44 in 1249ms, sequenceid=472, compaction requested=true 2024-11-07T14:16:12,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 758cb99e0ef7c32067829f2f2f90dc44: 2024-11-07T14:16:12,243 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:12,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:12,243 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:12,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:12,243 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. because compaction request was cancelled 2024-11-07T14:16:12,243 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:A 2024-11-07T14:16:12,243 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. after waiting 0 ms 2024-11-07T14:16:12,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:12,243 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. because compaction request was cancelled 2024-11-07T14:16:12,243 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:12,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:12,244 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:B 2024-11-07T14:16:12,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 758cb99e0ef7c32067829f2f2f90dc44:C, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:12,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:12,244 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. because compaction request was cancelled 2024-11-07T14:16:12,244 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 758cb99e0ef7c32067829f2f2f90dc44:C 2024-11-07T14:16:12,244 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing 758cb99e0ef7c32067829f2f2f90dc44 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T14:16:12,246 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/fb6778030a904c9197418bb4bd3f7f53 is 50, key is test_row_1/A:col10/1730988971002/Put/seqid=0 2024-11-07T14:16:12,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742385_1561 (size=9857) 2024-11-07T14:16:12,650 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/fb6778030a904c9197418bb4bd3f7f53 2024-11-07T14:16:12,656 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/48a7c3a9ea6843b9ab7c8a217d758186 is 50, key is test_row_1/B:col10/1730988971002/Put/seqid=0 2024-11-07T14:16:12,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742386_1562 (size=9857) 2024-11-07T14:16:13,061 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/48a7c3a9ea6843b9ab7c8a217d758186 2024-11-07T14:16:13,072 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6e77ca52b22740edbd86b29cad937054 is 50, key is test_row_1/C:col10/1730988971002/Put/seqid=0 2024-11-07T14:16:13,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742387_1563 (size=9857) 2024-11-07T14:16:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T14:16:13,476 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6e77ca52b22740edbd86b29cad937054 2024-11-07T14:16:13,480 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/A/fb6778030a904c9197418bb4bd3f7f53 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fb6778030a904c9197418bb4bd3f7f53 2024-11-07T14:16:13,483 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fb6778030a904c9197418bb4bd3f7f53, entries=100, sequenceid=478, filesize=9.6 K 2024-11-07T14:16:13,483 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/B/48a7c3a9ea6843b9ab7c8a217d758186 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/48a7c3a9ea6843b9ab7c8a217d758186 2024-11-07T14:16:13,486 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/48a7c3a9ea6843b9ab7c8a217d758186, entries=100, sequenceid=478, filesize=9.6 K 2024-11-07T14:16:13,487 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/.tmp/C/6e77ca52b22740edbd86b29cad937054 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6e77ca52b22740edbd86b29cad937054 2024-11-07T14:16:13,489 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6e77ca52b22740edbd86b29cad937054, entries=100, sequenceid=478, filesize=9.6 K 2024-11-07T14:16:13,501 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 758cb99e0ef7c32067829f2f2f90dc44 in 1257ms, sequenceid=478, compaction requested=true 2024-11-07T14:16:13,501 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1250fce944b440878e82a2131c8301a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/cd3c7bcf9bf1416cb49eabbe28422a22, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b809486875c140079bb2bbf450b5302f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fefc34a2c2124511aea8969324c48ab4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/042efc8e884342698b868406a0a8bd31, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/ecf6da4d9dcf49ba85555914156a3510, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3fee9fa402704cd48772fd0ff809a05d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/121c88fa5abc401b95f2642b31481d6b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/8589fd14f03049c4885412ac6985450d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/4110da3ca21b486aab61e1ca42566f6f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/51bfe53914714813b491f370263fb55c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/43560e01d9334b0588c9188371a016b6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/db63701d89dc45dfb2057b7d082e0ae8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/5e56f4e00d1d4bb6acc7885b9dfbd62d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/041471a7ff33445b8289a8599f239860, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/772ed65beb244f2bafe4563684c10700, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/41a3542414a949b096fd5c85ad006e76, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2e9a54e4e2364083a20305099b7d1173, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b9b4b59a3f4b497a94d1c120992407f8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/f4ca6fbb5ef24aee946bedd1bf5ca6d5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/e12dade043214eb8afe93f07d9053fc0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1cef0eae35fc4f7ba6c62e168d4d82fa, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/28a26d40f31e411fbbe55f2afe9ec9d4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/a9138a439dd14b0bafcec80c4fb2a56a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/638a7c9732994fa98615a910ff797088, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/0a02085408eb46f5be9229c1df15df03, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2f93bb3bc7494a9d9c9f07c8f40fae0c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/83d082608d744ebea712d4a6770758b4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/138f8eb5a3634b5f996211edbb8206d9] to archive 2024-11-07T14:16:13,502 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:16:13,503 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1250fce944b440878e82a2131c8301a7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1250fce944b440878e82a2131c8301a7 2024-11-07T14:16:13,504 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/cd3c7bcf9bf1416cb49eabbe28422a22 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/cd3c7bcf9bf1416cb49eabbe28422a22 2024-11-07T14:16:13,505 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b809486875c140079bb2bbf450b5302f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b809486875c140079bb2bbf450b5302f 2024-11-07T14:16:13,506 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fefc34a2c2124511aea8969324c48ab4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fefc34a2c2124511aea8969324c48ab4 2024-11-07T14:16:13,507 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/042efc8e884342698b868406a0a8bd31 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/042efc8e884342698b868406a0a8bd31 2024-11-07T14:16:13,507 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/ecf6da4d9dcf49ba85555914156a3510 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/ecf6da4d9dcf49ba85555914156a3510 2024-11-07T14:16:13,508 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3fee9fa402704cd48772fd0ff809a05d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3fee9fa402704cd48772fd0ff809a05d 2024-11-07T14:16:13,508 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/121c88fa5abc401b95f2642b31481d6b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/121c88fa5abc401b95f2642b31481d6b 2024-11-07T14:16:13,509 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/8589fd14f03049c4885412ac6985450d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/8589fd14f03049c4885412ac6985450d 2024-11-07T14:16:13,510 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/4110da3ca21b486aab61e1ca42566f6f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/4110da3ca21b486aab61e1ca42566f6f 2024-11-07T14:16:13,510 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/51bfe53914714813b491f370263fb55c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/51bfe53914714813b491f370263fb55c 2024-11-07T14:16:13,511 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/43560e01d9334b0588c9188371a016b6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/43560e01d9334b0588c9188371a016b6 2024-11-07T14:16:13,511 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/db63701d89dc45dfb2057b7d082e0ae8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/db63701d89dc45dfb2057b7d082e0ae8 2024-11-07T14:16:13,512 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/5e56f4e00d1d4bb6acc7885b9dfbd62d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/5e56f4e00d1d4bb6acc7885b9dfbd62d 2024-11-07T14:16:13,513 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/041471a7ff33445b8289a8599f239860 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/041471a7ff33445b8289a8599f239860 2024-11-07T14:16:13,513 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/772ed65beb244f2bafe4563684c10700 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/772ed65beb244f2bafe4563684c10700 2024-11-07T14:16:13,514 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/41a3542414a949b096fd5c85ad006e76 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/41a3542414a949b096fd5c85ad006e76 2024-11-07T14:16:13,515 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2e9a54e4e2364083a20305099b7d1173 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2e9a54e4e2364083a20305099b7d1173 2024-11-07T14:16:13,515 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b9b4b59a3f4b497a94d1c120992407f8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/b9b4b59a3f4b497a94d1c120992407f8 2024-11-07T14:16:13,516 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/f4ca6fbb5ef24aee946bedd1bf5ca6d5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/f4ca6fbb5ef24aee946bedd1bf5ca6d5 2024-11-07T14:16:13,516 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/e12dade043214eb8afe93f07d9053fc0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/e12dade043214eb8afe93f07d9053fc0 2024-11-07T14:16:13,517 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1cef0eae35fc4f7ba6c62e168d4d82fa to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/1cef0eae35fc4f7ba6c62e168d4d82fa 2024-11-07T14:16:13,518 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/28a26d40f31e411fbbe55f2afe9ec9d4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/28a26d40f31e411fbbe55f2afe9ec9d4 2024-11-07T14:16:13,519 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/a9138a439dd14b0bafcec80c4fb2a56a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/a9138a439dd14b0bafcec80c4fb2a56a 2024-11-07T14:16:13,519 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/638a7c9732994fa98615a910ff797088 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/638a7c9732994fa98615a910ff797088 2024-11-07T14:16:13,520 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/0a02085408eb46f5be9229c1df15df03 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/0a02085408eb46f5be9229c1df15df03 2024-11-07T14:16:13,520 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2f93bb3bc7494a9d9c9f07c8f40fae0c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/2f93bb3bc7494a9d9c9f07c8f40fae0c 2024-11-07T14:16:13,521 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/83d082608d744ebea712d4a6770758b4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/83d082608d744ebea712d4a6770758b4 2024-11-07T14:16:13,522 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/138f8eb5a3634b5f996211edbb8206d9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/138f8eb5a3634b5f996211edbb8206d9 2024-11-07T14:16:13,523 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/acdf09be141c40188b308f4b86be9aa1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a2b3bc3075344f868b09b079868110a4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e808f2e864ea4055aec383c1e23fe7d0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/433322143a0440bb84260d650ba8c461, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cac82e2989704a9b9276538989e6fcf8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/8728d202da6e45b5a50caaae5b0a9eb1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f38f34c7d83344d69ba0d67b03350f78, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/82bf139a434e4d0a8d5368feda2b01a6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b9ccbc41c54478991d22c51c51e926f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/988e5436fd254d21b30c068dc5cff3d6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f42581c6b44a4a549ae0d7fb345800e2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b752a08c853844caac91013c1892deaf, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b7aa6235ef0b41a8b1176ac644ab9b1d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/fa72e73614ed46a3ab32f166a1da659e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/3a44b85367cf41ebafe727bbe3457bf0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/dce5ecfdab3e4f399473080d1a69c0e0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/6065d09d41384b08a1c96e848b8559f8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d7124e3ebd2e47c98d627b4a3aceb89d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/73af2ee080c743279e75302411131fbb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/95c8983b15664cb4a82cb8740da18119, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cef1c84383894a4a9dcc6280efd1b4c3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/bc66bb311b174b72bff5dd3d7afa7104, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/4dd16e069941482c823a6671bb92af58, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d3878913a39b411c950fdfc991c30569, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0d91adc76694df49b2b70e9b80bcdc5, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/9b978e21bf24401d869384f249f123de, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e9250eda711a44b1a05db30b33c8fc05, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/312a42a6d4424bf19b0be507f8453534, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/0d45403db52e4b1e8843cfa1466fc344] to archive 2024-11-07T14:16:13,524 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:16:13,525 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/acdf09be141c40188b308f4b86be9aa1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/acdf09be141c40188b308f4b86be9aa1 2024-11-07T14:16:13,526 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a2b3bc3075344f868b09b079868110a4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a2b3bc3075344f868b09b079868110a4 2024-11-07T14:16:13,526 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e808f2e864ea4055aec383c1e23fe7d0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e808f2e864ea4055aec383c1e23fe7d0 2024-11-07T14:16:13,527 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/433322143a0440bb84260d650ba8c461 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/433322143a0440bb84260d650ba8c461 2024-11-07T14:16:13,528 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cac82e2989704a9b9276538989e6fcf8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cac82e2989704a9b9276538989e6fcf8 2024-11-07T14:16:13,528 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/8728d202da6e45b5a50caaae5b0a9eb1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/8728d202da6e45b5a50caaae5b0a9eb1 2024-11-07T14:16:13,529 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f38f34c7d83344d69ba0d67b03350f78 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f38f34c7d83344d69ba0d67b03350f78 2024-11-07T14:16:13,530 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/82bf139a434e4d0a8d5368feda2b01a6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/82bf139a434e4d0a8d5368feda2b01a6 2024-11-07T14:16:13,530 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b9ccbc41c54478991d22c51c51e926f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b9ccbc41c54478991d22c51c51e926f 2024-11-07T14:16:13,531 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/988e5436fd254d21b30c068dc5cff3d6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/988e5436fd254d21b30c068dc5cff3d6 2024-11-07T14:16:13,532 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f42581c6b44a4a549ae0d7fb345800e2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/f42581c6b44a4a549ae0d7fb345800e2 2024-11-07T14:16:13,532 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b752a08c853844caac91013c1892deaf to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b752a08c853844caac91013c1892deaf 2024-11-07T14:16:13,533 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b7aa6235ef0b41a8b1176ac644ab9b1d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/b7aa6235ef0b41a8b1176ac644ab9b1d 2024-11-07T14:16:13,534 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/fa72e73614ed46a3ab32f166a1da659e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/fa72e73614ed46a3ab32f166a1da659e 2024-11-07T14:16:13,534 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/3a44b85367cf41ebafe727bbe3457bf0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/3a44b85367cf41ebafe727bbe3457bf0 2024-11-07T14:16:13,535 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/dce5ecfdab3e4f399473080d1a69c0e0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/dce5ecfdab3e4f399473080d1a69c0e0 2024-11-07T14:16:13,536 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/6065d09d41384b08a1c96e848b8559f8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/6065d09d41384b08a1c96e848b8559f8 2024-11-07T14:16:13,537 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d7124e3ebd2e47c98d627b4a3aceb89d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d7124e3ebd2e47c98d627b4a3aceb89d 2024-11-07T14:16:13,537 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/73af2ee080c743279e75302411131fbb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/73af2ee080c743279e75302411131fbb 2024-11-07T14:16:13,538 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/95c8983b15664cb4a82cb8740da18119 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/95c8983b15664cb4a82cb8740da18119 2024-11-07T14:16:13,539 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cef1c84383894a4a9dcc6280efd1b4c3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/cef1c84383894a4a9dcc6280efd1b4c3 2024-11-07T14:16:13,539 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/bc66bb311b174b72bff5dd3d7afa7104 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/bc66bb311b174b72bff5dd3d7afa7104 2024-11-07T14:16:13,540 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/4dd16e069941482c823a6671bb92af58 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/4dd16e069941482c823a6671bb92af58 2024-11-07T14:16:13,541 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d3878913a39b411c950fdfc991c30569 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d3878913a39b411c950fdfc991c30569 2024-11-07T14:16:13,542 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0d91adc76694df49b2b70e9b80bcdc5 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0d91adc76694df49b2b70e9b80bcdc5 2024-11-07T14:16:13,542 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/9b978e21bf24401d869384f249f123de to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/9b978e21bf24401d869384f249f123de 2024-11-07T14:16:13,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e9250eda711a44b1a05db30b33c8fc05 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/e9250eda711a44b1a05db30b33c8fc05 2024-11-07T14:16:13,544 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/312a42a6d4424bf19b0be507f8453534 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/312a42a6d4424bf19b0be507f8453534 2024-11-07T14:16:13,545 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/0d45403db52e4b1e8843cfa1466fc344 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/0d45403db52e4b1e8843cfa1466fc344 2024-11-07T14:16:13,546 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/4079f92164024697979d8579919f8830, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/68593edc23384d1e8512e8cffc697caa, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/83af349e4f1c4b1b8e5c5c8ce5ee7203, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e703c12f36ff4716976b46c6ad80c788, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a7ffec13725b4a049ad90676cd26df85, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fedd7c1333ed474b9289000091db404c, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/f9569a635f144c37a63dbf55dce43ad4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/df28565a49f247e78ee567e5e13076b1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/2b19a4ba96074d99b83cb5ccb3a5c7ce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/5ce7d0ce14894bb5b5fcb115577833e0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dfddb67b669d4d1d9355681e6621afdd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cd0e25112f3644d6bbdf39abdf0a1494, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/20f3b35c8ad447dabeaf59238bf5f099, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cb185434cd0a469fbee66115c39769e4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/42fb2fbdd9784956bf0a4e9053adc351, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/73c7595d6dc24fc6b91a77280ec2c396, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/41c61bf516d9459f89503b49304abdf8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/d386ae0a536548488284b88d98046919, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3ec1e01abac6475f9dd08492b92b6ec1, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e51ba4f48771443794a6def80ce0b36e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6dbdcb190412476280369b607d7d28dc, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/21024b9a38764ab3a89fc0e09531f195, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3b2437fb262f4007beba1ec555f8ff46, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/0bc2b568c9de402287e7fe5f6d1d8037, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e2b7bb4d7c3b4ef1928347a0a734b089, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fe45dba2a65d4fe5b61e204f37ffd48e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/79d07f4a226342a48114361a6d57d3b6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dc0871e7dc1f42ef85f0a190432f8a02, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a1ac36710a7a417189103c075a11c9fb] to archive 2024-11-07T14:16:13,547 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:16:13,548 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/4079f92164024697979d8579919f8830 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/4079f92164024697979d8579919f8830 2024-11-07T14:16:13,548 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/68593edc23384d1e8512e8cffc697caa to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/68593edc23384d1e8512e8cffc697caa 2024-11-07T14:16:13,549 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/83af349e4f1c4b1b8e5c5c8ce5ee7203 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/83af349e4f1c4b1b8e5c5c8ce5ee7203 2024-11-07T14:16:13,550 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e703c12f36ff4716976b46c6ad80c788 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e703c12f36ff4716976b46c6ad80c788 2024-11-07T14:16:13,550 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a7ffec13725b4a049ad90676cd26df85 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a7ffec13725b4a049ad90676cd26df85 2024-11-07T14:16:13,551 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fedd7c1333ed474b9289000091db404c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fedd7c1333ed474b9289000091db404c 2024-11-07T14:16:13,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/f9569a635f144c37a63dbf55dce43ad4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/f9569a635f144c37a63dbf55dce43ad4 2024-11-07T14:16:13,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/df28565a49f247e78ee567e5e13076b1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/df28565a49f247e78ee567e5e13076b1 2024-11-07T14:16:13,553 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/2b19a4ba96074d99b83cb5ccb3a5c7ce to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/2b19a4ba96074d99b83cb5ccb3a5c7ce 2024-11-07T14:16:13,553 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/5ce7d0ce14894bb5b5fcb115577833e0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/5ce7d0ce14894bb5b5fcb115577833e0 2024-11-07T14:16:13,554 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dfddb67b669d4d1d9355681e6621afdd to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dfddb67b669d4d1d9355681e6621afdd 2024-11-07T14:16:13,555 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cd0e25112f3644d6bbdf39abdf0a1494 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cd0e25112f3644d6bbdf39abdf0a1494 2024-11-07T14:16:13,555 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/20f3b35c8ad447dabeaf59238bf5f099 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/20f3b35c8ad447dabeaf59238bf5f099 2024-11-07T14:16:13,556 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cb185434cd0a469fbee66115c39769e4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cb185434cd0a469fbee66115c39769e4 2024-11-07T14:16:13,556 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/42fb2fbdd9784956bf0a4e9053adc351 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/42fb2fbdd9784956bf0a4e9053adc351 2024-11-07T14:16:13,557 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/73c7595d6dc24fc6b91a77280ec2c396 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/73c7595d6dc24fc6b91a77280ec2c396 2024-11-07T14:16:13,558 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/41c61bf516d9459f89503b49304abdf8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/41c61bf516d9459f89503b49304abdf8 2024-11-07T14:16:13,558 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/d386ae0a536548488284b88d98046919 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/d386ae0a536548488284b88d98046919 2024-11-07T14:16:13,559 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3ec1e01abac6475f9dd08492b92b6ec1 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3ec1e01abac6475f9dd08492b92b6ec1 2024-11-07T14:16:13,559 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e51ba4f48771443794a6def80ce0b36e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e51ba4f48771443794a6def80ce0b36e 2024-11-07T14:16:13,560 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6dbdcb190412476280369b607d7d28dc to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6dbdcb190412476280369b607d7d28dc 2024-11-07T14:16:13,561 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/21024b9a38764ab3a89fc0e09531f195 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/21024b9a38764ab3a89fc0e09531f195 2024-11-07T14:16:13,561 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3b2437fb262f4007beba1ec555f8ff46 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/3b2437fb262f4007beba1ec555f8ff46 2024-11-07T14:16:13,562 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/0bc2b568c9de402287e7fe5f6d1d8037 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/0bc2b568c9de402287e7fe5f6d1d8037 2024-11-07T14:16:13,563 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e2b7bb4d7c3b4ef1928347a0a734b089 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/e2b7bb4d7c3b4ef1928347a0a734b089 2024-11-07T14:16:13,564 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fe45dba2a65d4fe5b61e204f37ffd48e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/fe45dba2a65d4fe5b61e204f37ffd48e 2024-11-07T14:16:13,565 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6b53c4aa9b5440eaa3aa5e7eb56ffddc 2024-11-07T14:16:13,566 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/79d07f4a226342a48114361a6d57d3b6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/79d07f4a226342a48114361a6d57d3b6 2024-11-07T14:16:13,566 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dc0871e7dc1f42ef85f0a190432f8a02 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/dc0871e7dc1f42ef85f0a190432f8a02 2024-11-07T14:16:13,567 DEBUG [StoreCloser-TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a1ac36710a7a417189103c075a11c9fb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/a1ac36710a7a417189103c075a11c9fb 2024-11-07T14:16:13,570 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/recovered.edits/481.seqid, newMaxSeqId=481, maxSeqId=1 2024-11-07T14:16:13,571 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. 2024-11-07T14:16:13,571 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 758cb99e0ef7c32067829f2f2f90dc44: Waiting for close lock at 1730988971168Running coprocessor pre-close hooks at 1730988971168Disabling compacts and flushes for region at 1730988971168Disabling writes for close at 1730988972243 (+1075 ms)Obtaining lock to block concurrent updates at 1730988972244 (+1 ms)Preparing flush snapshotting stores in 758cb99e0ef7c32067829f2f2f90dc44 at 1730988972244Finished memstore snapshotting TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730988972244Flushing stores of TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44. at 1730988972244Flushing 758cb99e0ef7c32067829f2f2f90dc44/A: creating writer at 1730988972244Flushing 758cb99e0ef7c32067829f2f2f90dc44/A: appending metadata at 1730988972246 (+2 ms)Flushing 758cb99e0ef7c32067829f2f2f90dc44/A: closing flushed file at 1730988972246Flushing 758cb99e0ef7c32067829f2f2f90dc44/B: creating writer at 1730988972654 (+408 ms)Flushing 758cb99e0ef7c32067829f2f2f90dc44/B: appending metadata at 1730988972656 (+2 ms)Flushing 758cb99e0ef7c32067829f2f2f90dc44/B: closing flushed file at 1730988972656Flushing 758cb99e0ef7c32067829f2f2f90dc44/C: creating writer at 1730988973068 (+412 ms)Flushing 758cb99e0ef7c32067829f2f2f90dc44/C: appending metadata at 1730988973072 (+4 ms)Flushing 758cb99e0ef7c32067829f2f2f90dc44/C: closing flushed file at 1730988973072Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6638c997: reopening flushed file at 1730988973479 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20fbc169: reopening flushed file at 1730988973483 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c783de1: reopening flushed file at 1730988973486 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 758cb99e0ef7c32067829f2f2f90dc44 in 1257ms, sequenceid=478, compaction requested=true at 1730988973501 (+15 ms)Writing region close event to WAL at 1730988973568 (+67 ms)Running coprocessor post-close hooks at 1730988973571 (+3 ms)Closed at 1730988973571 2024-11-07T14:16:13,572 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:13,573 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=758cb99e0ef7c32067829f2f2f90dc44, regionState=CLOSED 2024-11-07T14:16:13,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:16:13,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-07T14:16:13,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 758cb99e0ef7c32067829f2f2f90dc44, server=b7cc9d764de6,37737,1730988826985 in 2.5600 sec 2024-11-07T14:16:13,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-07T14:16:13,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=758cb99e0ef7c32067829f2f2f90dc44, UNASSIGN in 2.5640 sec 2024-11-07T14:16:13,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-07T14:16:13,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5670 sec 2024-11-07T14:16:13,580 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988973580"}]},"ts":"1730988973580"} 2024-11-07T14:16:13,581 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T14:16:13,581 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T14:16:13,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5750 sec 2024-11-07T14:16:15,077 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:16:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T14:16:15,173 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:15,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T14:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:15,175 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:15,176 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-07T14:16:15,178 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:15,180 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/recovered.edits] 2024-11-07T14:16:15,182 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/104321e31d704c77b39bc2ab8e9aa2e6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/104321e31d704c77b39bc2ab8e9aa2e6 2024-11-07T14:16:15,183 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3819bb3119da496fb6401c2fe91deb68 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/3819bb3119da496fb6401c2fe91deb68 2024-11-07T14:16:15,184 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/591124bf762148f38ea0656925e7ef66 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/591124bf762148f38ea0656925e7ef66 2024-11-07T14:16:15,184 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/c2b637471ec448ad913bc8d661d9eaf6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/c2b637471ec448ad913bc8d661d9eaf6 2024-11-07T14:16:15,185 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fb6778030a904c9197418bb4bd3f7f53 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/A/fb6778030a904c9197418bb4bd3f7f53 2024-11-07T14:16:15,187 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b424f5f998341b6ada804398e8b0c52 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/2b424f5f998341b6ada804398e8b0c52 2024-11-07T14:16:15,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/48a7c3a9ea6843b9ab7c8a217d758186 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/48a7c3a9ea6843b9ab7c8a217d758186 2024-11-07T14:16:15,189 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/557cc868b3fe498385d9832a2c30a205 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/557cc868b3fe498385d9832a2c30a205 2024-11-07T14:16:15,189 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a004f1c44f4f46fab24b752e7d944ec2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/a004f1c44f4f46fab24b752e7d944ec2 2024-11-07T14:16:15,190 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0b2a4af83db48d08330e42ea62fd1d6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/B/d0b2a4af83db48d08330e42ea62fd1d6 2024-11-07T14:16:15,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/14b7f2c68d0443aa92cb5b7ca043921d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/14b7f2c68d0443aa92cb5b7ca043921d 2024-11-07T14:16:15,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6e77ca52b22740edbd86b29cad937054 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/6e77ca52b22740edbd86b29cad937054 2024-11-07T14:16:15,193 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/b65f1b3c48a248f6ac32756f743ad167 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/b65f1b3c48a248f6ac32756f743ad167 2024-11-07T14:16:15,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cc08873bc88c4784b994a57e4d00f933 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/C/cc08873bc88c4784b994a57e4d00f933 2024-11-07T14:16:15,197 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/recovered.edits/481.seqid to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44/recovered.edits/481.seqid 2024-11-07T14:16:15,197 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/758cb99e0ef7c32067829f2f2f90dc44 2024-11-07T14:16:15,197 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T14:16:15,200 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:15,202 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T14:16:15,204 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T14:16:15,205 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:15,205 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T14:16:15,206 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730988975205"}]},"ts":"9223372036854775807"} 2024-11-07T14:16:15,208 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T14:16:15,208 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 758cb99e0ef7c32067829f2f2f90dc44, NAME => 'TestAcidGuarantees,,1730988947984.758cb99e0ef7c32067829f2f2f90dc44.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T14:16:15,208 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T14:16:15,208 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730988975208"}]},"ts":"9223372036854775807"} 2024-11-07T14:16:15,210 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T14:16:15,211 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:15,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-07T14:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-07T14:16:15,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T14:16:15,293 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:15,299 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=243 (was 242) - Thread LEAK? -, OpenFileDescriptor=461 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=590 (was 566) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=294 (was 590) 2024-11-07T14:16:15,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=243, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=590, ProcessCount=11, AvailableMemoryMB=293 2024-11-07T14:16:15,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:16:15,309 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:16:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:15,311 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T14:16:15,311 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-07T14:16:15,311 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T14:16:15,312 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T14:16:15,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742388_1564 (size=959) 2024-11-07T14:16:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T14:16:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T14:16:15,718 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9 2024-11-07T14:16:15,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742389_1565 (size=53) 2024-11-07T14:16:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T14:16:16,125 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:16:16,125 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 57f705175086737d129131015f71452d, disabling compactions & flushes 2024-11-07T14:16:16,125 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:16,125 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:16,125 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. after waiting 0 ms 2024-11-07T14:16:16,125 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:16,125 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:16,125 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 57f705175086737d129131015f71452d: Waiting for close lock at 1730988976125Disabling compacts and flushes for region at 1730988976125Disabling writes for close at 1730988976125Writing region close event to WAL at 1730988976125Closed at 1730988976125 2024-11-07T14:16:16,126 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T14:16:16,127 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730988976127"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730988976127"}]},"ts":"1730988976127"} 2024-11-07T14:16:16,128 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T14:16:16,129 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T14:16:16,129 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988976129"}]},"ts":"1730988976129"} 2024-11-07T14:16:16,130 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T14:16:16,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, ASSIGN}] 2024-11-07T14:16:16,131 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, ASSIGN 2024-11-07T14:16:16,132 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, ASSIGN; state=OFFLINE, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=false 2024-11-07T14:16:16,283 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=57f705175086737d129131015f71452d, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:16,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, ASSIGN because future has completed 2024-11-07T14:16:16,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:16:16,439 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:16,440 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:16:16,440 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 57f705175086737d129131015f71452d 2024-11-07T14:16:16,440 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:16:16,440 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for 57f705175086737d129131015f71452d 2024-11-07T14:16:16,440 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for 57f705175086737d129131015f71452d 2024-11-07T14:16:16,441 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 57f705175086737d129131015f71452d 2024-11-07T14:16:16,442 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57f705175086737d129131015f71452d columnFamilyName A 2024-11-07T14:16:16,442 DEBUG [StoreOpener-57f705175086737d129131015f71452d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:16,442 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(327): Store=57f705175086737d129131015f71452d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:16:16,442 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 57f705175086737d129131015f71452d 2024-11-07T14:16:16,443 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57f705175086737d129131015f71452d columnFamilyName B 2024-11-07T14:16:16,443 DEBUG [StoreOpener-57f705175086737d129131015f71452d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:16,444 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(327): Store=57f705175086737d129131015f71452d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:16:16,444 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 57f705175086737d129131015f71452d 2024-11-07T14:16:16,445 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57f705175086737d129131015f71452d columnFamilyName C 2024-11-07T14:16:16,445 DEBUG [StoreOpener-57f705175086737d129131015f71452d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:16,445 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(327): Store=57f705175086737d129131015f71452d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:16:16,445 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for 57f705175086737d129131015f71452d 2024-11-07T14:16:16,446 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d 2024-11-07T14:16:16,446 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d 2024-11-07T14:16:16,447 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for 57f705175086737d129131015f71452d 2024-11-07T14:16:16,447 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for 57f705175086737d129131015f71452d 2024-11-07T14:16:16,447 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:16:16,448 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for 57f705175086737d129131015f71452d 2024-11-07T14:16:16,449 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T14:16:16,449 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened 57f705175086737d129131015f71452d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68739060, jitterRate=0.024291813373565674}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:16:16,449 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57f705175086737d129131015f71452d 2024-11-07T14:16:16,450 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for 57f705175086737d129131015f71452d: Running coprocessor pre-open hook at 1730988976440Writing region info on filesystem at 1730988976440Initializing all the Stores at 1730988976441 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988976441Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988976441Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988976441Cleaning up temporary data from old regions at 1730988976447 (+6 ms)Running coprocessor post-open hooks at 1730988976449 (+2 ms)Region opened successfully at 1730988976450 (+1 ms) 2024-11-07T14:16:16,451 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., pid=154, masterSystemTime=1730988976437 2024-11-07T14:16:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T14:16:16,452 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:16,452 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:16,453 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=57f705175086737d129131015f71452d, regionState=OPEN, openSeqNum=2, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:16,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:16:16,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-07T14:16:16,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 in 169 msec 2024-11-07T14:16:16,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-07T14:16:16,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, ASSIGN in 325 msec 2024-11-07T14:16:16,457 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T14:16:16,458 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730988976458"}]},"ts":"1730988976458"} 2024-11-07T14:16:16,459 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T14:16:16,460 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T14:16:16,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-07T14:16:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T14:16:17,463 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:17,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaec12e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:17,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:17,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:17,469 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310f2610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:17,471 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e657d9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:17,473 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:17,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:17,475 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T14:16:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b7cc9d764de6,40745,1730988826115 2024-11-07T14:16:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c56bfce 2024-11-07T14:16:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T14:16:17,477 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T14:16:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T14:16:17,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T14:16:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:17,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742390_1566 (size=995) 2024-11-07T14:16:17,896 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T14:16:17,896 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T14:16:17,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:16:17,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, REOPEN/MOVE}] 2024-11-07T14:16:17,907 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, REOPEN/MOVE 2024-11-07T14:16:17,908 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=57f705175086737d129131015f71452d, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:17,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, REOPEN/MOVE because future has completed 2024-11-07T14:16:17,909 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:16:17,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:16:18,062 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close 57f705175086737d129131015f71452d 2024-11-07T14:16:18,062 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:16:18,062 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing 57f705175086737d129131015f71452d, disabling compactions & flushes 2024-11-07T14:16:18,062 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,062 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,062 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. after waiting 0 ms 2024-11-07T14:16:18,062 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,065 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T14:16:18,066 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,066 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for 57f705175086737d129131015f71452d: Waiting for close lock at 1730988978062Running coprocessor pre-close hooks at 1730988978062Disabling compacts and flushes for region at 1730988978062Disabling writes for close at 1730988978062Writing region close event to WAL at 1730988978062Running coprocessor post-close hooks at 1730988978066 (+4 ms)Closed at 1730988978066 2024-11-07T14:16:18,066 WARN [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: 57f705175086737d129131015f71452d to self. 2024-11-07T14:16:18,068 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed 57f705175086737d129131015f71452d 2024-11-07T14:16:18,068 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=57f705175086737d129131015f71452d, regionState=CLOSED 2024-11-07T14:16:18,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:16:18,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-07T14:16:18,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 in 161 msec 2024-11-07T14:16:18,072 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, REOPEN/MOVE; state=CLOSED, location=b7cc9d764de6,37737,1730988826985; forceNewPlan=false, retain=true 2024-11-07T14:16:18,222 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=57f705175086737d129131015f71452d, regionState=OPENING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, REOPEN/MOVE because future has completed 2024-11-07T14:16:18,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:16:18,378 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,378 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T14:16:18,379 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 57f705175086737d129131015f71452d 2024-11-07T14:16:18,379 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T14:16:18,379 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for 57f705175086737d129131015f71452d 2024-11-07T14:16:18,379 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for 57f705175086737d129131015f71452d 2024-11-07T14:16:18,380 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 57f705175086737d129131015f71452d 2024-11-07T14:16:18,380 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57f705175086737d129131015f71452d columnFamilyName A 2024-11-07T14:16:18,381 DEBUG [StoreOpener-57f705175086737d129131015f71452d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:18,382 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(327): Store=57f705175086737d129131015f71452d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:16:18,382 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 57f705175086737d129131015f71452d 2024-11-07T14:16:18,383 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57f705175086737d129131015f71452d columnFamilyName B 2024-11-07T14:16:18,383 DEBUG [StoreOpener-57f705175086737d129131015f71452d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:18,383 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(327): Store=57f705175086737d129131015f71452d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:16:18,383 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 57f705175086737d129131015f71452d 2024-11-07T14:16:18,384 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57f705175086737d129131015f71452d columnFamilyName C 2024-11-07T14:16:18,384 DEBUG [StoreOpener-57f705175086737d129131015f71452d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:18,384 INFO [StoreOpener-57f705175086737d129131015f71452d-1 {}] regionserver.HStore(327): Store=57f705175086737d129131015f71452d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T14:16:18,384 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for 57f705175086737d129131015f71452d 2024-11-07T14:16:18,385 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d 2024-11-07T14:16:18,385 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d 2024-11-07T14:16:18,386 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for 57f705175086737d129131015f71452d 2024-11-07T14:16:18,386 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for 57f705175086737d129131015f71452d 2024-11-07T14:16:18,386 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T14:16:18,387 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for 57f705175086737d129131015f71452d 2024-11-07T14:16:18,388 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened 57f705175086737d129131015f71452d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74758623, jitterRate=0.11399029195308685}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T14:16:18,388 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57f705175086737d129131015f71452d 2024-11-07T14:16:18,388 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for 57f705175086737d129131015f71452d: Running coprocessor pre-open hook at 1730988978379Writing region info on filesystem at 1730988978379Initializing all the Stores at 1730988978379Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988978379Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988978380 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730988978380Cleaning up temporary data from old regions at 1730988978386 (+6 ms)Running coprocessor post-open hooks at 1730988978388 (+2 ms)Region opened successfully at 1730988978388 2024-11-07T14:16:18,389 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., pid=159, masterSystemTime=1730988978376 2024-11-07T14:16:18,390 DEBUG [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,390 INFO [RS_OPEN_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,390 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=57f705175086737d129131015f71452d, regionState=OPEN, openSeqNum=5, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:16:18,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-07T14:16:18,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 in 168 msec 2024-11-07T14:16:18,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-07T14:16:18,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, REOPEN/MOVE in 488 msec 2024-11-07T14:16:18,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-07T14:16:18,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-07T14:16:18,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 917 msec 2024-11-07T14:16:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-07T14:16:18,399 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:18,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d1887e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,401 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ed0249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,402 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33507cdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2a1e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,404 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6886d730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,405 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b3b38e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdeed38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,407 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773e49a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,408 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558ee211, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6eabbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1112a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd716b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3be2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,412 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbee24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f84aa30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d7368f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,415 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@374a1056, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@274b199b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108b0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,417 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d673f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e9f6d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5be7c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,421 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf4dc32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,422 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ecc2629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162f1ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,424 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563d2d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cafdae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e62e123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b7cc9d764de6,40745,-1 for getting cluster id 2024-11-07T14:16:18,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T14:16:18,427 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '17ff3ed6-b733-4bf6-b059-c68920fd597e' 2024-11-07T14:16:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T14:16:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "17ff3ed6-b733-4bf6-b059-c68920fd597e" 2024-11-07T14:16:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65dc703c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b7cc9d764de6,40745,-1] 2024-11-07T14:16:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T14:16:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:18,428 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T14:16:18,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461d087d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T14:16:18,429 DEBUG [Thread-2510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,429 DEBUG [Thread-2529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,429 DEBUG [Thread-2514 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,429 DEBUG [Thread-2512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:18,430 DEBUG [Thread-2521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,430 DEBUG [Thread-2518 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-07T14:16:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,431 DEBUG [Thread-2516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,431 DEBUG [Thread-2527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T14:16:18,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,433 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:18,433 DEBUG [Thread-2523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,434 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:18,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,436 DEBUG [Thread-2525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T14:16:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b7cc9d764de6,37737,1730988826985, seqNum=-1] 2024-11-07T14:16:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T14:16:18,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T14:16:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5] 2024-11-07T14:16:18,441 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:18,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T14:16:18,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ddb50898ecae47f98792c4a6db895072_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988978437/Put/seqid=0 2024-11-07T14:16:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989038461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989038466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989038467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742391_1567 (size=14594) 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989038467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989038471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T14:16:18,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989038572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989038583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989038582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989038583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989038583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:18,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T14:16:18,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989038781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989038792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989038792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989038792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989038792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:18,870 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:18,873 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ddb50898ecae47f98792c4a6db895072_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ddb50898ecae47f98792c4a6db895072_57f705175086737d129131015f71452d 2024-11-07T14:16:18,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/a41d2251585c4aa1ba437a19852a3d65, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:18,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/a41d2251585c4aa1ba437a19852a3d65 is 175, key is test_row_0/A:col10/1730988978437/Put/seqid=0 2024-11-07T14:16:18,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742392_1568 (size=39549) 2024-11-07T14:16:18,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:18,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:18,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T14:16:19,094 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T14:16:19,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989039092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989039102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989039102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989039102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989039104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,278 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/a41d2251585c4aa1ba437a19852a3d65 2024-11-07T14:16:19,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/b737df9c01fc498989acea9710b69f4d is 50, key is test_row_0/B:col10/1730988978437/Put/seqid=0 2024-11-07T14:16:19,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742393_1569 (size=12001) 2024-11-07T14:16:19,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/b737df9c01fc498989acea9710b69f4d 2024-11-07T14:16:19,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/476b21b1164442d7b1684bd708779a23 is 50, key is test_row_0/C:col10/1730988978437/Put/seqid=0 2024-11-07T14:16:19,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742394_1570 (size=12001) 2024-11-07T14:16:19,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:19,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:19,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T14:16:19,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989039602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989039612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989039612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989039612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989039612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:19,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:19,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/476b21b1164442d7b1684bd708779a23 2024-11-07T14:16:19,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/a41d2251585c4aa1ba437a19852a3d65 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/a41d2251585c4aa1ba437a19852a3d65 2024-11-07T14:16:19,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/a41d2251585c4aa1ba437a19852a3d65, entries=200, sequenceid=17, filesize=38.6 K 2024-11-07T14:16:19,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/b737df9c01fc498989acea9710b69f4d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b737df9c01fc498989acea9710b69f4d 2024-11-07T14:16:19,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b737df9c01fc498989acea9710b69f4d, entries=150, sequenceid=17, filesize=11.7 K 2024-11-07T14:16:19,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/476b21b1164442d7b1684bd708779a23 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/476b21b1164442d7b1684bd708779a23 2024-11-07T14:16:19,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/476b21b1164442d7b1684bd708779a23, entries=150, sequenceid=17, filesize=11.7 K 2024-11-07T14:16:19,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 57f705175086737d129131015f71452d in 1315ms, sequenceid=17, compaction requested=false 2024-11-07T14:16:19,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:19,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T14:16:19,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:19,816 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T14:16:19,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107317974be12b0434eb4c98eac7f4b9a57_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988978465/Put/seqid=0 2024-11-07T14:16:19,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742395_1571 (size=12154) 2024-11-07T14:16:20,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:20,241 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107317974be12b0434eb4c98eac7f4b9a57_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107317974be12b0434eb4c98eac7f4b9a57_57f705175086737d129131015f71452d 2024-11-07T14:16:20,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/2c6d4378b3e346279e7067a9019ee33b, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:20,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/2c6d4378b3e346279e7067a9019ee33b is 175, key is test_row_0/A:col10/1730988978465/Put/seqid=0 2024-11-07T14:16:20,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742396_1572 (size=30955) 2024-11-07T14:16:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T14:16:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:20,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989040631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989040631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989040632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989040632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,636 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 57f705175086737d129131015f71452d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:16:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989040632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,636 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 57f705175086737d129131015f71452d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,655 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/2c6d4378b3e346279e7067a9019ee33b 2024-11-07T14:16:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/25b2a7bac3c048889f3ffdfe33f4c821 is 50, key is test_row_0/B:col10/1730988978465/Put/seqid=0 2024-11-07T14:16:20,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742397_1573 (size=12001) 2024-11-07T14:16:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989040741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989040742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989040742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989040952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989040952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989040952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:20,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/25b2a7bac3c048889f3ffdfe33f4c821 2024-11-07T14:16:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/8a7325e3ff604e209123d4d5714fbd30 is 50, key is test_row_0/C:col10/1730988978465/Put/seqid=0 2024-11-07T14:16:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742398_1574 (size=12001) 2024-11-07T14:16:21,081 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/8a7325e3ff604e209123d4d5714fbd30 2024-11-07T14:16:21,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/2c6d4378b3e346279e7067a9019ee33b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/2c6d4378b3e346279e7067a9019ee33b 2024-11-07T14:16:21,090 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/2c6d4378b3e346279e7067a9019ee33b, entries=150, sequenceid=41, filesize=30.2 K 2024-11-07T14:16:21,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/25b2a7bac3c048889f3ffdfe33f4c821 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/25b2a7bac3c048889f3ffdfe33f4c821 2024-11-07T14:16:21,094 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/25b2a7bac3c048889f3ffdfe33f4c821, entries=150, sequenceid=41, filesize=11.7 K 2024-11-07T14:16:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/8a7325e3ff604e209123d4d5714fbd30 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/8a7325e3ff604e209123d4d5714fbd30 2024-11-07T14:16:21,098 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/8a7325e3ff604e209123d4d5714fbd30, entries=150, sequenceid=41, filesize=11.7 K 2024-11-07T14:16:21,099 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 57f705175086737d129131015f71452d in 1283ms, sequenceid=41, compaction requested=false 2024-11-07T14:16:21,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:21,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:21,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-07T14:16:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-07T14:16:21,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-07T14:16:21,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6660 sec 2024-11-07T14:16:21,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.6720 sec 2024-11-07T14:16:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:21,265 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:21,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T14:16:21,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074a27fa0e149e491ea6ae9bc0a20a70be_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988980629/Put/seqid=0 2024-11-07T14:16:21,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742399_1575 (size=14594) 2024-11-07T14:16:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989041294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989041295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989041296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989041402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989041402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989041412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989041611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989041612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989041622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:21,676 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074a27fa0e149e491ea6ae9bc0a20a70be_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074a27fa0e149e491ea6ae9bc0a20a70be_57f705175086737d129131015f71452d 2024-11-07T14:16:21,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/272686f37fd64d1c82aa367ecb2de5b2, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:21,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/272686f37fd64d1c82aa367ecb2de5b2 is 175, key is test_row_0/A:col10/1730988980629/Put/seqid=0 2024-11-07T14:16:21,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742400_1576 (size=39549) 2024-11-07T14:16:21,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989041922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989041922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989041932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:22,082 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/272686f37fd64d1c82aa367ecb2de5b2 2024-11-07T14:16:22,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/1cd92882b30a438dbf83ebc0e2ae723f is 50, key is test_row_0/B:col10/1730988980629/Put/seqid=0 2024-11-07T14:16:22,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742401_1577 (size=12001) 2024-11-07T14:16:22,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989042432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989042432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:22,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989042442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:22,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/1cd92882b30a438dbf83ebc0e2ae723f 2024-11-07T14:16:22,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/2613ae80bccb40198c6224a2324542e8 is 50, key is test_row_0/C:col10/1730988980629/Put/seqid=0 2024-11-07T14:16:22,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742402_1578 (size=12001) 2024-11-07T14:16:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T14:16:22,592 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:22,593 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-07T14:16:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T14:16:22,595 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:22,596 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:22,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989042652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:22,654 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 57f705175086737d129131015f71452d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:16:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:22,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989042662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:22,665 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 57f705175086737d129131015f71452d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:16:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T14:16:22,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T14:16:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:22,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:22,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T14:16:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:22,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:22,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T14:16:22,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/2613ae80bccb40198c6224a2324542e8 2024-11-07T14:16:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/272686f37fd64d1c82aa367ecb2de5b2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/272686f37fd64d1c82aa367ecb2de5b2 2024-11-07T14:16:22,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/272686f37fd64d1c82aa367ecb2de5b2, entries=200, sequenceid=56, filesize=38.6 K 2024-11-07T14:16:22,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/1cd92882b30a438dbf83ebc0e2ae723f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1cd92882b30a438dbf83ebc0e2ae723f 2024-11-07T14:16:22,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1cd92882b30a438dbf83ebc0e2ae723f, entries=150, sequenceid=56, filesize=11.7 K 2024-11-07T14:16:22,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/2613ae80bccb40198c6224a2324542e8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/2613ae80bccb40198c6224a2324542e8 2024-11-07T14:16:22,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/2613ae80bccb40198c6224a2324542e8, entries=150, sequenceid=56, filesize=11.7 K 2024-11-07T14:16:22,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 57f705175086737d129131015f71452d in 1685ms, sequenceid=56, compaction requested=true 2024-11-07T14:16:22,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:22,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:16:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T14:16:22,954 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:22,954 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:22,959 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:22,959 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:22,959 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/C is initiating minor compaction (all files) 2024-11-07T14:16:22,959 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/A is initiating minor compaction (all files) 2024-11-07T14:16:22,959 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/C in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:22,959 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/476b21b1164442d7b1684bd708779a23, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/8a7325e3ff604e209123d4d5714fbd30, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/2613ae80bccb40198c6224a2324542e8] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=35.2 K 2024-11-07T14:16:22,960 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 476b21b1164442d7b1684bd708779a23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730988978437 2024-11-07T14:16:22,960 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a7325e3ff604e209123d4d5714fbd30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730988978459 2024-11-07T14:16:22,961 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 2613ae80bccb40198c6224a2324542e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988980629 2024-11-07T14:16:22,962 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/A in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:22,963 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/a41d2251585c4aa1ba437a19852a3d65, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/2c6d4378b3e346279e7067a9019ee33b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/272686f37fd64d1c82aa367ecb2de5b2] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=107.5 K 2024-11-07T14:16:22,963 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:22,963 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/a41d2251585c4aa1ba437a19852a3d65, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/2c6d4378b3e346279e7067a9019ee33b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/272686f37fd64d1c82aa367ecb2de5b2] 2024-11-07T14:16:22,964 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting a41d2251585c4aa1ba437a19852a3d65, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730988978437 2024-11-07T14:16:22,965 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c6d4378b3e346279e7067a9019ee33b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730988978459 2024-11-07T14:16:22,966 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272686f37fd64d1c82aa367ecb2de5b2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988980629 2024-11-07T14:16:22,979 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#C#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:22,979 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/42b8052971484f748a2e1452402f5046 is 50, key is test_row_0/C:col10/1730988980629/Put/seqid=0 2024-11-07T14:16:22,991 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:23,011 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411070b6b6bfe7fd549df963e6abdfaa7dbe4_57f705175086737d129131015f71452d store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:23,013 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411070b6b6bfe7fd549df963e6abdfaa7dbe4_57f705175086737d129131015f71452d, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:23,013 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070b6b6bfe7fd549df963e6abdfaa7dbe4_57f705175086737d129131015f71452d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:23,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742403_1579 (size=12104) 2024-11-07T14:16:23,040 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/42b8052971484f748a2e1452402f5046 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/42b8052971484f748a2e1452402f5046 2024-11-07T14:16:23,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742404_1580 (size=4469) 2024-11-07T14:16:23,046 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#A#compaction#500 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:23,047 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/58c9cb18aeb240c4aabdbdba509beece is 175, key is test_row_0/A:col10/1730988980629/Put/seqid=0 2024-11-07T14:16:23,050 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/C of 57f705175086737d129131015f71452d into 42b8052971484f748a2e1452402f5046(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:23,050 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:23,050 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/C, priority=13, startTime=1730988982951; duration=0sec 2024-11-07T14:16:23,050 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:23,050 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:C 2024-11-07T14:16:23,050 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:23,053 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:23,053 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/B is initiating minor compaction (all files) 2024-11-07T14:16:23,053 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/B in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:23,053 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b737df9c01fc498989acea9710b69f4d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/25b2a7bac3c048889f3ffdfe33f4c821, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1cd92882b30a438dbf83ebc0e2ae723f] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=35.2 K 2024-11-07T14:16:23,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742405_1581 (size=31058) 2024-11-07T14:16:23,054 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b737df9c01fc498989acea9710b69f4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730988978437 2024-11-07T14:16:23,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T14:16:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:23,060 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T14:16:23,063 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 25b2a7bac3c048889f3ffdfe33f4c821, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730988978459 2024-11-07T14:16:23,063 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd92882b30a438dbf83ebc0e2ae723f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988980629 2024-11-07T14:16:23,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107251ff5e71ae9472b938005ed83932eeb_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988981295/Put/seqid=0 2024-11-07T14:16:23,073 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/58c9cb18aeb240c4aabdbdba509beece as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/58c9cb18aeb240c4aabdbdba509beece 2024-11-07T14:16:23,080 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:23,081 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/b3d11906f98d414b98064ca6f9194867 is 50, key is test_row_0/B:col10/1730988980629/Put/seqid=0 2024-11-07T14:16:23,083 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/A of 57f705175086737d129131015f71452d into 58c9cb18aeb240c4aabdbdba509beece(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:23,083 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:23,083 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/A, priority=13, startTime=1730988982950; duration=0sec 2024-11-07T14:16:23,084 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:23,084 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:A 2024-11-07T14:16:23,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742406_1582 (size=12154) 2024-11-07T14:16:23,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:23,093 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107251ff5e71ae9472b938005ed83932eeb_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107251ff5e71ae9472b938005ed83932eeb_57f705175086737d129131015f71452d 2024-11-07T14:16:23,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bd32b067be0c48e4bfcfa4fe806b4660, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:23,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bd32b067be0c48e4bfcfa4fe806b4660 is 175, key is test_row_0/A:col10/1730988981295/Put/seqid=0 2024-11-07T14:16:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742407_1583 (size=12104) 2024-11-07T14:16:23,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742408_1584 (size=30955) 2024-11-07T14:16:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T14:16:23,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:23,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989043486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989043487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989043492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,527 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bd32b067be0c48e4bfcfa4fe806b4660 2024-11-07T14:16:23,544 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/b3d11906f98d414b98064ca6f9194867 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b3d11906f98d414b98064ca6f9194867 2024-11-07T14:16:23,551 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/B of 57f705175086737d129131015f71452d into b3d11906f98d414b98064ca6f9194867(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:23,551 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:23,551 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/B, priority=13, startTime=1730988982951; duration=0sec 2024-11-07T14:16:23,551 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:23,551 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:B 2024-11-07T14:16:23,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/067fd40b46d44dbda5e4b7f28f662c2e is 50, key is test_row_0/B:col10/1730988981295/Put/seqid=0 2024-11-07T14:16:23,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742409_1585 (size=12001) 2024-11-07T14:16:23,599 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/067fd40b46d44dbda5e4b7f28f662c2e 2024-11-07T14:16:23,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989043602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989043602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989043602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/1f8eb895faf74451acf1e93bef5ba389 is 50, key is test_row_0/C:col10/1730988981295/Put/seqid=0 2024-11-07T14:16:23,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742410_1586 (size=12001) 2024-11-07T14:16:23,611 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/1f8eb895faf74451acf1e93bef5ba389 2024-11-07T14:16:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bd32b067be0c48e4bfcfa4fe806b4660 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd32b067be0c48e4bfcfa4fe806b4660 2024-11-07T14:16:23,621 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd32b067be0c48e4bfcfa4fe806b4660, entries=150, sequenceid=78, filesize=30.2 K 2024-11-07T14:16:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/067fd40b46d44dbda5e4b7f28f662c2e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/067fd40b46d44dbda5e4b7f28f662c2e 2024-11-07T14:16:23,642 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/067fd40b46d44dbda5e4b7f28f662c2e, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T14:16:23,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/1f8eb895faf74451acf1e93bef5ba389 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/1f8eb895faf74451acf1e93bef5ba389 2024-11-07T14:16:23,648 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/1f8eb895faf74451acf1e93bef5ba389, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T14:16:23,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 57f705175086737d129131015f71452d in 589ms, sequenceid=78, compaction requested=false 2024-11-07T14:16:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-07T14:16:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-07T14:16:23,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-07T14:16:23,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0540 sec 2024-11-07T14:16:23,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0600 sec 2024-11-07T14:16:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T14:16:23,733 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:23,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-07T14:16:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T14:16:23,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:23,736 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:23,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:23,820 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:23,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T14:16:23,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110774029fd1f8c4455b9f604fead37e93fd_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988983815/Put/seqid=0 2024-11-07T14:16:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T14:16:23,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742411_1587 (size=14594) 2024-11-07T14:16:23,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989043859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,872 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:23,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989043868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989043871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,878 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110774029fd1f8c4455b9f604fead37e93fd_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774029fd1f8c4455b9f604fead37e93fd_57f705175086737d129131015f71452d 2024-11-07T14:16:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/c883e037dbd24fd69d54f61a20f0e1c2, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:23,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/c883e037dbd24fd69d54f61a20f0e1c2 is 175, key is test_row_0/A:col10/1730988983815/Put/seqid=0 2024-11-07T14:16:23,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T14:16:23,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:23,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:23,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742412_1588 (size=39549) 2024-11-07T14:16:23,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/c883e037dbd24fd69d54f61a20f0e1c2 2024-11-07T14:16:23,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/c771a64ac53441cb811db91501dd7c22 is 50, key is test_row_0/B:col10/1730988983815/Put/seqid=0 2024-11-07T14:16:23,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742413_1589 (size=12001) 2024-11-07T14:16:23,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989043983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989043983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989043987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T14:16:24,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T14:16:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989044192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989044192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989044203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:24,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T14:16:24,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:24,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T14:16:24,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T14:16:24,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/c771a64ac53441cb811db91501dd7c22 2024-11-07T14:16:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/677c42bfa5394c22a916fe7e22884b11 is 50, key is test_row_0/C:col10/1730988983815/Put/seqid=0 2024-11-07T14:16:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742414_1590 (size=12001) 2024-11-07T14:16:24,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989044503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:24,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989044503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:24,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989044519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:24,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T14:16:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T14:16:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/677c42bfa5394c22a916fe7e22884b11 2024-11-07T14:16:24,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/c883e037dbd24fd69d54f61a20f0e1c2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c883e037dbd24fd69d54f61a20f0e1c2 2024-11-07T14:16:24,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c883e037dbd24fd69d54f61a20f0e1c2, entries=200, sequenceid=98, filesize=38.6 K 2024-11-07T14:16:24,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T14:16:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/c771a64ac53441cb811db91501dd7c22 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c771a64ac53441cb811db91501dd7c22 2024-11-07T14:16:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:24,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c771a64ac53441cb811db91501dd7c22, entries=150, sequenceid=98, filesize=11.7 K 2024-11-07T14:16:24,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/677c42bfa5394c22a916fe7e22884b11 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/677c42bfa5394c22a916fe7e22884b11 2024-11-07T14:16:24,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/677c42bfa5394c22a916fe7e22884b11, entries=150, sequenceid=98, filesize=11.7 K 2024-11-07T14:16:24,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 57f705175086737d129131015f71452d in 1033ms, sequenceid=98, compaction requested=true 2024-11-07T14:16:24,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:24,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:24,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:24,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:24,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:16:24,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:24,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T14:16:24,854 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:24,854 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:24,855 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:24,855 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:24,855 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/A is initiating minor compaction (all files) 2024-11-07T14:16:24,855 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/C is initiating minor compaction (all files) 2024-11-07T14:16:24,855 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/A in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,855 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/C in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,855 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/58c9cb18aeb240c4aabdbdba509beece, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd32b067be0c48e4bfcfa4fe806b4660, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c883e037dbd24fd69d54f61a20f0e1c2] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=99.2 K 2024-11-07T14:16:24,855 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/42b8052971484f748a2e1452402f5046, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/1f8eb895faf74451acf1e93bef5ba389, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/677c42bfa5394c22a916fe7e22884b11] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=35.3 K 2024-11-07T14:16:24,855 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,855 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/58c9cb18aeb240c4aabdbdba509beece, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd32b067be0c48e4bfcfa4fe806b4660, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c883e037dbd24fd69d54f61a20f0e1c2] 2024-11-07T14:16:24,855 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b8052971484f748a2e1452402f5046, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988980629 2024-11-07T14:16:24,855 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c9cb18aeb240c4aabdbdba509beece, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988980629 2024-11-07T14:16:24,855 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f8eb895faf74451acf1e93bef5ba389, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730988981288 2024-11-07T14:16:24,856 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd32b067be0c48e4bfcfa4fe806b4660, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730988981288 2024-11-07T14:16:24,857 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 677c42bfa5394c22a916fe7e22884b11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730988983484 2024-11-07T14:16:24,857 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c883e037dbd24fd69d54f61a20f0e1c2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730988983484 2024-11-07T14:16:24,867 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T14:16:24,872 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:24,873 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/46bbec01bbba4a1f814df5d6fd0616ce is 50, key is test_row_0/C:col10/1730988983815/Put/seqid=0 2024-11-07T14:16:24,879 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411079fe5e1b9b62f492f91a4e257b2934f35_57f705175086737d129131015f71452d store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:24,881 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411079fe5e1b9b62f492f91a4e257b2934f35_57f705175086737d129131015f71452d, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:24,881 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079fe5e1b9b62f492f91a4e257b2934f35_57f705175086737d129131015f71452d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:24,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742415_1591 (size=12207) 2024-11-07T14:16:24,915 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/46bbec01bbba4a1f814df5d6fd0616ce as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/46bbec01bbba4a1f814df5d6fd0616ce 2024-11-07T14:16:24,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742416_1592 (size=4469) 2024-11-07T14:16:24,921 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/C of 57f705175086737d129131015f71452d into 46bbec01bbba4a1f814df5d6fd0616ce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:24,922 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:24,922 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/C, priority=13, startTime=1730988984854; duration=0sec 2024-11-07T14:16:24,922 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:24,922 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:C 2024-11-07T14:16:24,922 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:24,923 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:24,923 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/B is initiating minor compaction (all files) 2024-11-07T14:16:24,923 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/B in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,923 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b3d11906f98d414b98064ca6f9194867, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/067fd40b46d44dbda5e4b7f28f662c2e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c771a64ac53441cb811db91501dd7c22] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=35.3 K 2024-11-07T14:16:24,923 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d11906f98d414b98064ca6f9194867, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1730988980629 2024-11-07T14:16:24,923 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 067fd40b46d44dbda5e4b7f28f662c2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730988981288 2024-11-07T14:16:24,924 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c771a64ac53441cb811db91501dd7c22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730988983484 2024-11-07T14:16:24,929 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:24,929 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/c90986a35a7147b9a07960433fa4be4b is 50, key is test_row_0/B:col10/1730988983815/Put/seqid=0 2024-11-07T14:16:24,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742417_1593 (size=12207) 2024-11-07T14:16:24,940 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/c90986a35a7147b9a07960433fa4be4b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c90986a35a7147b9a07960433fa4be4b 2024-11-07T14:16:24,949 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/B of 57f705175086737d129131015f71452d into c90986a35a7147b9a07960433fa4be4b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:24,949 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:24,949 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/B, priority=13, startTime=1730988984854; duration=0sec 2024-11-07T14:16:24,949 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:24,949 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:B 2024-11-07T14:16:24,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T14:16:24,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:24,983 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:16:25,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ca49b0de296347ae92f3ff0c04a56bcd_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988983856/Put/seqid=0 2024-11-07T14:16:25,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742418_1594 (size=12154) 2024-11-07T14:16:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:25,011 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ca49b0de296347ae92f3ff0c04a56bcd_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca49b0de296347ae92f3ff0c04a56bcd_57f705175086737d129131015f71452d 2024-11-07T14:16:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/720123dd9ed2489d864266acb17d71bd, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/720123dd9ed2489d864266acb17d71bd is 175, key is test_row_0/A:col10/1730988983856/Put/seqid=0 2024-11-07T14:16:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:25,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742419_1595 (size=30955) 2024-11-07T14:16:25,018 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/720123dd9ed2489d864266acb17d71bd 2024-11-07T14:16:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/6fa8cc8efdb441a3956d26fe66816be8 is 50, key is test_row_0/B:col10/1730988983856/Put/seqid=0 2024-11-07T14:16:25,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742420_1596 (size=12001) 2024-11-07T14:16:25,047 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/6fa8cc8efdb441a3956d26fe66816be8 2024-11-07T14:16:25,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9a3b76a5f6ed43d49194d8d902a9a67d is 50, key is test_row_0/C:col10/1730988983856/Put/seqid=0 2024-11-07T14:16:25,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989045054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989045058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989045060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742421_1597 (size=12001) 2024-11-07T14:16:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989045172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989045172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989045172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,318 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#A#compaction#508 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:25,319 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bd4ce2a1fc284d04a32c0aae564c573d is 175, key is test_row_0/A:col10/1730988983815/Put/seqid=0 2024-11-07T14:16:25,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742422_1598 (size=31161) 2024-11-07T14:16:25,328 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bd4ce2a1fc284d04a32c0aae564c573d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd4ce2a1fc284d04a32c0aae564c573d 2024-11-07T14:16:25,333 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/A of 57f705175086737d129131015f71452d into bd4ce2a1fc284d04a32c0aae564c573d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:25,334 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:25,334 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/A, priority=13, startTime=1730988984853; duration=0sec 2024-11-07T14:16:25,334 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:25,334 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:A 2024-11-07T14:16:25,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989045382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989045383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989045387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,468 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9a3b76a5f6ed43d49194d8d902a9a67d 2024-11-07T14:16:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/720123dd9ed2489d864266acb17d71bd as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/720123dd9ed2489d864266acb17d71bd 2024-11-07T14:16:25,475 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/720123dd9ed2489d864266acb17d71bd, entries=150, sequenceid=118, filesize=30.2 K 2024-11-07T14:16:25,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/6fa8cc8efdb441a3956d26fe66816be8 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/6fa8cc8efdb441a3956d26fe66816be8 2024-11-07T14:16:25,478 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/6fa8cc8efdb441a3956d26fe66816be8, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T14:16:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9a3b76a5f6ed43d49194d8d902a9a67d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9a3b76a5f6ed43d49194d8d902a9a67d 2024-11-07T14:16:25,481 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9a3b76a5f6ed43d49194d8d902a9a67d, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T14:16:25,482 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 57f705175086737d129131015f71452d in 500ms, sequenceid=118, compaction requested=false 2024-11-07T14:16:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-07T14:16:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-07T14:16:25,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-07T14:16:25,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7470 sec 2024-11-07T14:16:25,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.7520 sec 2024-11-07T14:16:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:25,694 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:25,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T14:16:25,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072e79323087044a3b9912fd891264cfd7_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988985693/Put/seqid=0 2024-11-07T14:16:25,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742423_1599 (size=17234) 2024-11-07T14:16:25,724 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:25,727 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072e79323087044a3b9912fd891264cfd7_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072e79323087044a3b9912fd891264cfd7_57f705175086737d129131015f71452d 2024-11-07T14:16:25,727 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/c06d602bae17447283f24dcb65055c1e, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:25,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/c06d602bae17447283f24dcb65055c1e is 175, key is test_row_0/A:col10/1730988985693/Put/seqid=0 2024-11-07T14:16:25,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742424_1600 (size=48339) 2024-11-07T14:16:25,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989045730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989045730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989045730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989045842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989045842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989045842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T14:16:25,882 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:25,883 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-07T14:16:25,884 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T14:16:25,885 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:25,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T14:16:26,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T14:16:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989046052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989046052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989046052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/c06d602bae17447283f24dcb65055c1e 2024-11-07T14:16:26,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/83bfa0f5317e41649c46f91e462db2a6 is 50, key is test_row_0/B:col10/1730988985693/Put/seqid=0 2024-11-07T14:16:26,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T14:16:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:26,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T14:16:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:26,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742425_1601 (size=12151) 2024-11-07T14:16:26,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/83bfa0f5317e41649c46f91e462db2a6 2024-11-07T14:16:26,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b9880f070d044730a006f75b275c8dd7 is 50, key is test_row_0/C:col10/1730988985693/Put/seqid=0 2024-11-07T14:16:26,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742426_1602 (size=12151) 2024-11-07T14:16:26,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b9880f070d044730a006f75b275c8dd7 2024-11-07T14:16:26,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/c06d602bae17447283f24dcb65055c1e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c06d602bae17447283f24dcb65055c1e 2024-11-07T14:16:26,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c06d602bae17447283f24dcb65055c1e, entries=250, sequenceid=139, filesize=47.2 K 2024-11-07T14:16:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/83bfa0f5317e41649c46f91e462db2a6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/83bfa0f5317e41649c46f91e462db2a6 2024-11-07T14:16:26,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/83bfa0f5317e41649c46f91e462db2a6, entries=150, sequenceid=139, filesize=11.9 K 2024-11-07T14:16:26,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b9880f070d044730a006f75b275c8dd7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b9880f070d044730a006f75b275c8dd7 2024-11-07T14:16:26,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b9880f070d044730a006f75b275c8dd7, entries=150, sequenceid=139, filesize=11.9 K 2024-11-07T14:16:26,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for 57f705175086737d129131015f71452d in 572ms, sequenceid=139, compaction requested=true 2024-11-07T14:16:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:26,266 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:26,266 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:26,267 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:26,267 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:26,267 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/A is initiating minor compaction (all files) 2024-11-07T14:16:26,267 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/B is initiating minor compaction (all files) 2024-11-07T14:16:26,267 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/B in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,267 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/A in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,267 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c90986a35a7147b9a07960433fa4be4b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/6fa8cc8efdb441a3956d26fe66816be8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/83bfa0f5317e41649c46f91e462db2a6] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=35.5 K 2024-11-07T14:16:26,267 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd4ce2a1fc284d04a32c0aae564c573d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/720123dd9ed2489d864266acb17d71bd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c06d602bae17447283f24dcb65055c1e] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=107.9 K 2024-11-07T14:16:26,267 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,267 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd4ce2a1fc284d04a32c0aae564c573d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/720123dd9ed2489d864266acb17d71bd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c06d602bae17447283f24dcb65055c1e] 2024-11-07T14:16:26,268 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting c90986a35a7147b9a07960433fa4be4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730988983484 2024-11-07T14:16:26,268 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4ce2a1fc284d04a32c0aae564c573d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730988983484 2024-11-07T14:16:26,268 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 720123dd9ed2489d864266acb17d71bd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730988983838 2024-11-07T14:16:26,268 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fa8cc8efdb441a3956d26fe66816be8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730988983838 2024-11-07T14:16:26,268 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 83bfa0f5317e41649c46f91e462db2a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730988985055 2024-11-07T14:16:26,268 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting c06d602bae17447283f24dcb65055c1e, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730988985040 2024-11-07T14:16:26,274 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:26,275 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#B#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:26,275 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/12cf4a576e504243807e217103383f67 is 50, key is test_row_0/B:col10/1730988985693/Put/seqid=0 2024-11-07T14:16:26,286 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107bab66c48d1a34c20abe35622b5faf967_57f705175086737d129131015f71452d store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:26,289 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107bab66c48d1a34c20abe35622b5faf967_57f705175086737d129131015f71452d, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:26,289 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bab66c48d1a34c20abe35622b5faf967_57f705175086737d129131015f71452d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:26,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742427_1603 (size=12459) 2024-11-07T14:16:26,307 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/12cf4a576e504243807e217103383f67 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/12cf4a576e504243807e217103383f67 2024-11-07T14:16:26,312 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/B of 57f705175086737d129131015f71452d into 12cf4a576e504243807e217103383f67(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:26,312 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:26,313 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/B, priority=13, startTime=1730988986266; duration=0sec 2024-11-07T14:16:26,313 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:26,313 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:B 2024-11-07T14:16:26,313 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:26,319 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:26,319 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/C is initiating minor compaction (all files) 2024-11-07T14:16:26,319 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/C in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,319 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/46bbec01bbba4a1f814df5d6fd0616ce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9a3b76a5f6ed43d49194d8d902a9a67d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b9880f070d044730a006f75b275c8dd7] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=35.5 K 2024-11-07T14:16:26,320 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 46bbec01bbba4a1f814df5d6fd0616ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1730988983484 2024-11-07T14:16:26,320 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a3b76a5f6ed43d49194d8d902a9a67d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730988983838 2024-11-07T14:16:26,320 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b9880f070d044730a006f75b275c8dd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730988985055 2024-11-07T14:16:26,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742428_1604 (size=4469) 2024-11-07T14:16:26,326 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#C#compaction#519 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:26,327 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/09e7ed99fba04340a7beda8200cead0a is 50, key is test_row_0/C:col10/1730988985693/Put/seqid=0 2024-11-07T14:16:26,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742429_1605 (size=12459) 2024-11-07T14:16:26,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T14:16:26,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:26,355 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:16:26,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:26,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075570156bfe0a4d50ad8d293f6324c98c_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988985728/Put/seqid=0 2024-11-07T14:16:26,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742430_1606 (size=12304) 2024-11-07T14:16:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989046412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989046417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989046418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T14:16:26,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989046522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989046532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989046533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989046672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,674 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 57f705175086737d129131015f71452d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:16:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989046702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,704 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 57f705175086737d129131015f71452d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:16:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,725 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#A#compaction#518 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:26,725 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/89f15837114b49f68107d2ef7fc96d25 is 175, key is test_row_0/A:col10/1730988985693/Put/seqid=0 2024-11-07T14:16:26,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742431_1607 (size=31413) 2024-11-07T14:16:26,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989046735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,744 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/09e7ed99fba04340a7beda8200cead0a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/09e7ed99fba04340a7beda8200cead0a 2024-11-07T14:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989046742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,744 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/89f15837114b49f68107d2ef7fc96d25 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/89f15837114b49f68107d2ef7fc96d25 2024-11-07T14:16:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989046742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:26,748 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/C of 57f705175086737d129131015f71452d into 09e7ed99fba04340a7beda8200cead0a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:26,748 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:26,748 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/C, priority=13, startTime=1730988986266; duration=0sec 2024-11-07T14:16:26,748 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:26,748 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:C 2024-11-07T14:16:26,749 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/A of 57f705175086737d129131015f71452d into 89f15837114b49f68107d2ef7fc96d25(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:26,749 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:26,749 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/A, priority=13, startTime=1730988986266; duration=0sec 2024-11-07T14:16:26,749 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:26,749 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:A 2024-11-07T14:16:26,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:26,781 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075570156bfe0a4d50ad8d293f6324c98c_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075570156bfe0a4d50ad8d293f6324c98c_57f705175086737d129131015f71452d 2024-11-07T14:16:26,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/17b98ab3f47e4f89b0a51cedfae33f29, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:26,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/17b98ab3f47e4f89b0a51cedfae33f29 is 175, key is test_row_0/A:col10/1730988985728/Put/seqid=0 2024-11-07T14:16:26,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742432_1608 (size=31105) 2024-11-07T14:16:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T14:16:27,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989047042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989047052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989047052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,190 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/17b98ab3f47e4f89b0a51cedfae33f29 2024-11-07T14:16:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/08b1fb0ba160493390738f9effecb639 is 50, key is test_row_0/B:col10/1730988985728/Put/seqid=0 2024-11-07T14:16:27,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742433_1609 (size=12151) 2024-11-07T14:16:27,203 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/08b1fb0ba160493390738f9effecb639 2024-11-07T14:16:27,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/de9a0daaceaa4c4fa9982457075e173f is 50, key is test_row_0/C:col10/1730988985728/Put/seqid=0 2024-11-07T14:16:27,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742434_1610 (size=12151) 2024-11-07T14:16:27,228 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/de9a0daaceaa4c4fa9982457075e173f 2024-11-07T14:16:27,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/17b98ab3f47e4f89b0a51cedfae33f29 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/17b98ab3f47e4f89b0a51cedfae33f29 2024-11-07T14:16:27,243 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/17b98ab3f47e4f89b0a51cedfae33f29, entries=150, sequenceid=158, filesize=30.4 K 2024-11-07T14:16:27,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/08b1fb0ba160493390738f9effecb639 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/08b1fb0ba160493390738f9effecb639 2024-11-07T14:16:27,248 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/08b1fb0ba160493390738f9effecb639, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T14:16:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/de9a0daaceaa4c4fa9982457075e173f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/de9a0daaceaa4c4fa9982457075e173f 2024-11-07T14:16:27,254 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/de9a0daaceaa4c4fa9982457075e173f, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T14:16:27,259 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 57f705175086737d129131015f71452d in 903ms, sequenceid=158, compaction requested=false 2024-11-07T14:16:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-07T14:16:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-07T14:16:27,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-07T14:16:27,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3850 sec 2024-11-07T14:16:27,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.3940 sec 2024-11-07T14:16:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:27,554 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:27,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T14:16:27,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075d08b4ff2c4c4aa2aee4a737052100e2_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988986386/Put/seqid=0 2024-11-07T14:16:27,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742435_1611 (size=14794) 2024-11-07T14:16:27,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:27,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075d08b4ff2c4c4aa2aee4a737052100e2_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d08b4ff2c4c4aa2aee4a737052100e2_57f705175086737d129131015f71452d 2024-11-07T14:16:27,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/efc35a984ad3452a92f0fddf104b40b9, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:27,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/efc35a984ad3452a92f0fddf104b40b9 is 175, key is test_row_0/A:col10/1730988986386/Put/seqid=0 2024-11-07T14:16:27,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742436_1612 (size=39749) 2024-11-07T14:16:27,598 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/efc35a984ad3452a92f0fddf104b40b9 2024-11-07T14:16:27,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989047601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/583d8ef250224f86a9a02631df7b4df7 is 50, key is test_row_0/B:col10/1730988986386/Put/seqid=0 2024-11-07T14:16:27,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989047610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989047611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742437_1613 (size=12151) 2024-11-07T14:16:27,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989047712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989047722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989047723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989047922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989047932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989047932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T14:16:28,033 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:28,033 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-07T14:16:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T14:16:28,035 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:28,036 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:28,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:28,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/583d8ef250224f86a9a02631df7b4df7 2024-11-07T14:16:28,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/3a8b8b59816347aabb7ddaba04de9dd4 is 50, key is test_row_0/C:col10/1730988986386/Put/seqid=0 2024-11-07T14:16:28,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742438_1614 (size=12151) 2024-11-07T14:16:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T14:16:28,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T14:16:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:28,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989048232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989048243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989048243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T14:16:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T14:16:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:28,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/3a8b8b59816347aabb7ddaba04de9dd4 2024-11-07T14:16:28,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/efc35a984ad3452a92f0fddf104b40b9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/efc35a984ad3452a92f0fddf104b40b9 2024-11-07T14:16:28,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/efc35a984ad3452a92f0fddf104b40b9, entries=200, sequenceid=179, filesize=38.8 K 2024-11-07T14:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/583d8ef250224f86a9a02631df7b4df7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/583d8ef250224f86a9a02631df7b4df7 2024-11-07T14:16:28,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/583d8ef250224f86a9a02631df7b4df7, entries=150, sequenceid=179, filesize=11.9 K 2024-11-07T14:16:28,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/3a8b8b59816347aabb7ddaba04de9dd4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/3a8b8b59816347aabb7ddaba04de9dd4 2024-11-07T14:16:28,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/3a8b8b59816347aabb7ddaba04de9dd4, entries=150, sequenceid=179, filesize=11.9 K 2024-11-07T14:16:28,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=107.34 KB/109920 for 57f705175086737d129131015f71452d in 938ms, sequenceid=179, compaction requested=true 2024-11-07T14:16:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:28,493 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:28,493 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:28,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:28,494 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:28,494 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/B is initiating minor compaction (all files) 2024-11-07T14:16:28,494 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/B in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,494 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/12cf4a576e504243807e217103383f67, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/08b1fb0ba160493390738f9effecb639, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/583d8ef250224f86a9a02631df7b4df7] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=35.9 K 2024-11-07T14:16:28,494 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:28,494 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/A is initiating minor compaction (all files) 2024-11-07T14:16:28,494 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/A in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,495 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/89f15837114b49f68107d2ef7fc96d25, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/17b98ab3f47e4f89b0a51cedfae33f29, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/efc35a984ad3452a92f0fddf104b40b9] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=99.9 K 2024-11-07T14:16:28,495 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,495 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/89f15837114b49f68107d2ef7fc96d25, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/17b98ab3f47e4f89b0a51cedfae33f29, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/efc35a984ad3452a92f0fddf104b40b9] 2024-11-07T14:16:28,495 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 12cf4a576e504243807e217103383f67, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730988985055 2024-11-07T14:16:28,496 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f15837114b49f68107d2ef7fc96d25, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730988985055 2024-11-07T14:16:28,496 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b1fb0ba160493390738f9effecb639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730988985701 2024-11-07T14:16:28,496 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17b98ab3f47e4f89b0a51cedfae33f29, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730988985701 2024-11-07T14:16:28,496 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 583d8ef250224f86a9a02631df7b4df7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730988986386 2024-11-07T14:16:28,496 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc35a984ad3452a92f0fddf104b40b9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730988986386 2024-11-07T14:16:28,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-07T14:16:28,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,510 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T14:16:28,517 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#B#compaction#526 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:28,517 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/f7401e5812314ac7980f69df7f213137 is 50, key is test_row_0/B:col10/1730988986386/Put/seqid=0 2024-11-07T14:16:28,518 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b035dd8d1cd44bc1bf93883721ca100b_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988987593/Put/seqid=0 2024-11-07T14:16:28,543 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411074a7e0bc847ef423b88993f7b8cbe6785_57f705175086737d129131015f71452d store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:28,553 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411074a7e0bc847ef423b88993f7b8cbe6785_57f705175086737d129131015f71452d, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:28,554 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074a7e0bc847ef423b88993f7b8cbe6785_57f705175086737d129131015f71452d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:28,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742439_1615 (size=12561) 2024-11-07T14:16:28,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742440_1616 (size=12304) 2024-11-07T14:16:28,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:28,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b035dd8d1cd44bc1bf93883721ca100b_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b035dd8d1cd44bc1bf93883721ca100b_57f705175086737d129131015f71452d 2024-11-07T14:16:28,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/cb2a61111cf946208205ea8b49978dd6, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:28,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/cb2a61111cf946208205ea8b49978dd6 is 175, key is test_row_0/A:col10/1730988987593/Put/seqid=0 2024-11-07T14:16:28,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742441_1617 (size=4469) 2024-11-07T14:16:28,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742442_1618 (size=31105) 2024-11-07T14:16:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T14:16:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:28,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:28,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989048779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989048781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989048781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989048892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989048892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989048893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:28,969 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/f7401e5812314ac7980f69df7f213137 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/f7401e5812314ac7980f69df7f213137 2024-11-07T14:16:28,972 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/B of 57f705175086737d129131015f71452d into f7401e5812314ac7980f69df7f213137(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:28,973 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:28,973 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/B, priority=13, startTime=1730988988493; duration=0sec 2024-11-07T14:16:28,973 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:28,973 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:B 2024-11-07T14:16:28,973 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:28,974 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:28,974 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/C is initiating minor compaction (all files) 2024-11-07T14:16:28,974 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/C in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:28,974 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/09e7ed99fba04340a7beda8200cead0a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/de9a0daaceaa4c4fa9982457075e173f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/3a8b8b59816347aabb7ddaba04de9dd4] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=35.9 K 2024-11-07T14:16:28,974 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 09e7ed99fba04340a7beda8200cead0a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730988985055 2024-11-07T14:16:28,974 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting de9a0daaceaa4c4fa9982457075e173f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730988985701 2024-11-07T14:16:28,974 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a8b8b59816347aabb7ddaba04de9dd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730988986386 2024-11-07T14:16:28,980 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:28,980 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/113b3f1507ef43d2861a24901e54fa31 is 50, key is test_row_0/C:col10/1730988986386/Put/seqid=0 2024-11-07T14:16:28,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742443_1619 (size=12561) 2024-11-07T14:16:29,006 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#A#compaction#527 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:29,007 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/5239b862a8cb4ea9ad418e4614f19937 is 175, key is test_row_0/A:col10/1730988986386/Put/seqid=0 2024-11-07T14:16:29,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742444_1620 (size=31515) 2024-11-07T14:16:29,018 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/cb2a61111cf946208205ea8b49978dd6 2024-11-07T14:16:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/016920d959d44012ab57eb45c60b9e9d is 50, key is test_row_0/B:col10/1730988987593/Put/seqid=0 2024-11-07T14:16:29,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742445_1621 (size=12151) 2024-11-07T14:16:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989049112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989049114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989049114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T14:16:29,387 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/113b3f1507ef43d2861a24901e54fa31 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/113b3f1507ef43d2861a24901e54fa31 2024-11-07T14:16:29,391 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/C of 57f705175086737d129131015f71452d into 113b3f1507ef43d2861a24901e54fa31(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:29,391 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:29,392 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/C, priority=13, startTime=1730988988493; duration=0sec 2024-11-07T14:16:29,392 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:29,392 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:C 2024-11-07T14:16:29,422 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/5239b862a8cb4ea9ad418e4614f19937 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5239b862a8cb4ea9ad418e4614f19937 2024-11-07T14:16:29,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989049422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989049422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,430 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/A of 57f705175086737d129131015f71452d into 5239b862a8cb4ea9ad418e4614f19937(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:29,430 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:29,430 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/A, priority=13, startTime=1730988988493; duration=0sec 2024-11-07T14:16:29,430 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:29,430 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:A 2024-11-07T14:16:29,433 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/016920d959d44012ab57eb45c60b9e9d 2024-11-07T14:16:29,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989049432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/f714788da63541d2a67acacf062173a6 is 50, key is test_row_0/C:col10/1730988987593/Put/seqid=0 2024-11-07T14:16:29,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742446_1622 (size=12151) 2024-11-07T14:16:29,447 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/f714788da63541d2a67acacf062173a6 2024-11-07T14:16:29,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/cb2a61111cf946208205ea8b49978dd6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cb2a61111cf946208205ea8b49978dd6 2024-11-07T14:16:29,454 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cb2a61111cf946208205ea8b49978dd6, entries=150, sequenceid=198, filesize=30.4 K 2024-11-07T14:16:29,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/016920d959d44012ab57eb45c60b9e9d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/016920d959d44012ab57eb45c60b9e9d 2024-11-07T14:16:29,472 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/016920d959d44012ab57eb45c60b9e9d, entries=150, sequenceid=198, filesize=11.9 K 2024-11-07T14:16:29,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/f714788da63541d2a67acacf062173a6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/f714788da63541d2a67acacf062173a6 2024-11-07T14:16:29,478 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/f714788da63541d2a67acacf062173a6, entries=150, sequenceid=198, filesize=11.9 K 2024-11-07T14:16:29,479 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 57f705175086737d129131015f71452d in 970ms, sequenceid=198, compaction requested=false 2024-11-07T14:16:29,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:29,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:29,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-07T14:16:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-07T14:16:29,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-07T14:16:29,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-07T14:16:29,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.4520 sec 2024-11-07T14:16:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:29,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T14:16:29,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110770f0a34af90b4dcd8f4665f33ebe2ccc_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988989935/Put/seqid=0 2024-11-07T14:16:29,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742447_1623 (size=17284) 2024-11-07T14:16:29,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989049965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989049967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989049969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,975 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:29,977 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110770f0a34af90b4dcd8f4665f33ebe2ccc_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110770f0a34af90b4dcd8f4665f33ebe2ccc_57f705175086737d129131015f71452d 2024-11-07T14:16:29,978 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/9fa35368165d4c478f8fbacdfaeebc70, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:29,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/9fa35368165d4c478f8fbacdfaeebc70 is 175, key is test_row_0/A:col10/1730988989935/Put/seqid=0 2024-11-07T14:16:29,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742448_1624 (size=48389) 2024-11-07T14:16:30,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989050072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989050081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989050081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T14:16:30,183 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:30,183 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-07T14:16:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T14:16:30,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:30,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:30,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:30,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989050282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T14:16:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989050292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989050292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-07T14:16:30,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:30,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/9fa35368165d4c478f8fbacdfaeebc70 2024-11-07T14:16:30,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e0aba444fa444cd8a64e068fd113f264 is 50, key is test_row_0/B:col10/1730988989935/Put/seqid=0 2024-11-07T14:16:30,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742449_1625 (size=12151) 2024-11-07T14:16:30,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-07T14:16:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T14:16:30,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989050595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989050602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989050602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:30,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-07T14:16:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e0aba444fa444cd8a64e068fd113f264 2024-11-07T14:16:30,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-07T14:16:30,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:30,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T14:16:30,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/7e22d0c5d4264f6ba197f6f85f415ba0 is 50, key is test_row_0/C:col10/1730988989935/Put/seqid=0 2024-11-07T14:16:30,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742450_1626 (size=12151) 2024-11-07T14:16:30,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/7e22d0c5d4264f6ba197f6f85f415ba0 2024-11-07T14:16:30,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/9fa35368165d4c478f8fbacdfaeebc70 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/9fa35368165d4c478f8fbacdfaeebc70 2024-11-07T14:16:30,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/9fa35368165d4c478f8fbacdfaeebc70, entries=250, sequenceid=221, filesize=47.3 K 2024-11-07T14:16:30,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e0aba444fa444cd8a64e068fd113f264 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e0aba444fa444cd8a64e068fd113f264 2024-11-07T14:16:30,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e0aba444fa444cd8a64e068fd113f264, entries=150, sequenceid=221, filesize=11.9 K 2024-11-07T14:16:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/7e22d0c5d4264f6ba197f6f85f415ba0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/7e22d0c5d4264f6ba197f6f85f415ba0 2024-11-07T14:16:30,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/7e22d0c5d4264f6ba197f6f85f415ba0, entries=150, sequenceid=221, filesize=11.9 K 2024-11-07T14:16:30,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 57f705175086737d129131015f71452d in 921ms, sequenceid=221, compaction requested=true 2024-11-07T14:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T14:16:30,858 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:30,863 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:30,867 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:30,867 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/A is initiating minor compaction (all files) 2024-11-07T14:16:30,867 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/A in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,867 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5239b862a8cb4ea9ad418e4614f19937, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cb2a61111cf946208205ea8b49978dd6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/9fa35368165d4c478f8fbacdfaeebc70] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=108.4 K 2024-11-07T14:16:30,867 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,867 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5239b862a8cb4ea9ad418e4614f19937, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cb2a61111cf946208205ea8b49978dd6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/9fa35368165d4c478f8fbacdfaeebc70] 2024-11-07T14:16:30,875 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5239b862a8cb4ea9ad418e4614f19937, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730988986386 2024-11-07T14:16:30,876 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb2a61111cf946208205ea8b49978dd6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730988987576 2024-11-07T14:16:30,878 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:30,878 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fa35368165d4c478f8fbacdfaeebc70, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1730988988772 2024-11-07T14:16:30,878 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/C is initiating minor compaction (all files) 2024-11-07T14:16:30,878 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/C in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,878 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/113b3f1507ef43d2861a24901e54fa31, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/f714788da63541d2a67acacf062173a6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/7e22d0c5d4264f6ba197f6f85f415ba0] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=36.0 K 2024-11-07T14:16:30,879 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 113b3f1507ef43d2861a24901e54fa31, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730988986386 2024-11-07T14:16:30,880 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f714788da63541d2a67acacf062173a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730988987576 2024-11-07T14:16:30,880 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e22d0c5d4264f6ba197f6f85f415ba0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1730988988772 2024-11-07T14:16:30,885 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:30,896 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#C#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:30,897 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/63f84570003d458998a0b1abb9dd2282 is 50, key is test_row_0/C:col10/1730988989935/Put/seqid=0 2024-11-07T14:16:30,911 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411077379f4f8c438424a8771ff1fd890371f_57f705175086737d129131015f71452d store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:30,913 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411077379f4f8c438424a8771ff1fd890371f_57f705175086737d129131015f71452d, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:30,914 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077379f4f8c438424a8771ff1fd890371f_57f705175086737d129131015f71452d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:30,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-07T14:16:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,950 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T14:16:30,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742451_1627 (size=12663) 2024-11-07T14:16:30,962 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/63f84570003d458998a0b1abb9dd2282 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/63f84570003d458998a0b1abb9dd2282 2024-11-07T14:16:30,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742452_1628 (size=4469) 2024-11-07T14:16:30,965 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#A#compaction#535 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:30,966 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/faa86b443bd44993ae6add181c5b08da is 175, key is test_row_0/A:col10/1730988989935/Put/seqid=0 2024-11-07T14:16:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bb5238333ad0481ca7815dce152d9976_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988989963/Put/seqid=0 2024-11-07T14:16:30,969 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/C of 57f705175086737d129131015f71452d into 63f84570003d458998a0b1abb9dd2282(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:30,969 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:30,969 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/C, priority=13, startTime=1730988990857; duration=0sec 2024-11-07T14:16:30,970 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:30,970 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:C 2024-11-07T14:16:30,970 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:30,972 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:30,972 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/B is initiating minor compaction (all files) 2024-11-07T14:16:30,972 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/B in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:30,972 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/f7401e5812314ac7980f69df7f213137, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/016920d959d44012ab57eb45c60b9e9d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e0aba444fa444cd8a64e068fd113f264] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=36.0 K 2024-11-07T14:16:30,972 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting f7401e5812314ac7980f69df7f213137, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730988986386 2024-11-07T14:16:30,974 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 016920d959d44012ab57eb45c60b9e9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730988987576 2024-11-07T14:16:30,974 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e0aba444fa444cd8a64e068fd113f264, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1730988988772 2024-11-07T14:16:30,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742453_1629 (size=31617) 2024-11-07T14:16:30,984 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#B#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:30,984 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/1ecc2f2172f149b5bdb694a32535a7d3 is 50, key is test_row_0/B:col10/1730988989935/Put/seqid=0 2024-11-07T14:16:30,988 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/faa86b443bd44993ae6add181c5b08da as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/faa86b443bd44993ae6add181c5b08da 2024-11-07T14:16:30,996 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/A of 57f705175086737d129131015f71452d into faa86b443bd44993ae6add181c5b08da(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:30,996 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:30,996 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/A, priority=13, startTime=1730988990857; duration=0sec 2024-11-07T14:16:30,996 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:30,996 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:A 2024-11-07T14:16:30,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742454_1630 (size=12304) 2024-11-07T14:16:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:31,001 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bb5238333ad0481ca7815dce152d9976_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bb5238333ad0481ca7815dce152d9976_57f705175086737d129131015f71452d 2024-11-07T14:16:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/cade969da0454ca49449eb2edb1a906d, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/cade969da0454ca49449eb2edb1a906d is 175, key is test_row_0/A:col10/1730988989963/Put/seqid=0 2024-11-07T14:16:31,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742455_1631 (size=12663) 2024-11-07T14:16:31,013 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/1ecc2f2172f149b5bdb694a32535a7d3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1ecc2f2172f149b5bdb694a32535a7d3 2024-11-07T14:16:31,019 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/B of 57f705175086737d129131015f71452d into 1ecc2f2172f149b5bdb694a32535a7d3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:31,019 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:31,019 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/B, priority=13, startTime=1730988990857; duration=0sec 2024-11-07T14:16:31,019 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:31,019 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:B 2024-11-07T14:16:31,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742456_1632 (size=31105) 2024-11-07T14:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:31,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989051140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989051142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989051144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989051253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989051253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989051253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T14:16:31,439 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/cade969da0454ca49449eb2edb1a906d 2024-11-07T14:16:31,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e8829ced69ba4be7a9d133d28c6f1f30 is 50, key is test_row_0/B:col10/1730988989963/Put/seqid=0 2024-11-07T14:16:31,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742457_1633 (size=12151) 2024-11-07T14:16:31,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989051463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989051463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989051463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989051773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989051773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989051773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:31,858 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e8829ced69ba4be7a9d133d28c6f1f30 2024-11-07T14:16:31,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b03e55fa8288446286bb67f2a87e8e5f is 50, key is test_row_0/C:col10/1730988989963/Put/seqid=0 2024-11-07T14:16:31,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742458_1634 (size=12151) 2024-11-07T14:16:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989052282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989052282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989052282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:32,307 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b03e55fa8288446286bb67f2a87e8e5f 2024-11-07T14:16:32,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/cade969da0454ca49449eb2edb1a906d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cade969da0454ca49449eb2edb1a906d 2024-11-07T14:16:32,318 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cade969da0454ca49449eb2edb1a906d, entries=150, sequenceid=237, filesize=30.4 K 2024-11-07T14:16:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e8829ced69ba4be7a9d133d28c6f1f30 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e8829ced69ba4be7a9d133d28c6f1f30 2024-11-07T14:16:32,324 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e8829ced69ba4be7a9d133d28c6f1f30, entries=150, sequenceid=237, filesize=11.9 K 2024-11-07T14:16:32,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b03e55fa8288446286bb67f2a87e8e5f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b03e55fa8288446286bb67f2a87e8e5f 2024-11-07T14:16:32,329 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b03e55fa8288446286bb67f2a87e8e5f, entries=150, sequenceid=237, filesize=11.9 K 2024-11-07T14:16:32,329 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 57f705175086737d129131015f71452d in 1379ms, sequenceid=237, compaction requested=false 2024-11-07T14:16:32,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:32,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:32,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-07T14:16:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-07T14:16:32,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-07T14:16:32,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1460 sec 2024-11-07T14:16:32,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.1500 sec 2024-11-07T14:16:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-07T14:16:32,342 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:32,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-07T14:16:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T14:16:32,344 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:32,344 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:32,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T14:16:32,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-07T14:16:32,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:32,502 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T14:16:32,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f96c696b7e5b46e691ac086864867351_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988991139/Put/seqid=0 2024-11-07T14:16:32,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742459_1635 (size=12454) 2024-11-07T14:16:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T14:16:32,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:32,929 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f96c696b7e5b46e691ac086864867351_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f96c696b7e5b46e691ac086864867351_57f705175086737d129131015f71452d 2024-11-07T14:16:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/5518b4545168415c80132a3cb4cb8695, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/5518b4545168415c80132a3cb4cb8695 is 175, key is test_row_0/A:col10/1730988991139/Put/seqid=0 2024-11-07T14:16:32,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742460_1636 (size=31255) 2024-11-07T14:16:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T14:16:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:33,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:33,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989053316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989053318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989053324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,344 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/5518b4545168415c80132a3cb4cb8695 2024-11-07T14:16:33,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/916b86b22ffd46f495f8d4fdf5be08c0 is 50, key is test_row_0/B:col10/1730988991139/Put/seqid=0 2024-11-07T14:16:33,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742461_1637 (size=12301) 2024-11-07T14:16:33,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989053435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989053435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989053443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T14:16:33,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989053642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989053653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989053653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,830 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/916b86b22ffd46f495f8d4fdf5be08c0 2024-11-07T14:16:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/6aa7a99eefb648709d0dbf6c85a9ffcb is 50, key is test_row_0/C:col10/1730988991139/Put/seqid=0 2024-11-07T14:16:33,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742462_1638 (size=12301) 2024-11-07T14:16:33,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989053952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989053963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989053964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,283 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/6aa7a99eefb648709d0dbf6c85a9ffcb 2024-11-07T14:16:34,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/5518b4545168415c80132a3cb4cb8695 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5518b4545168415c80132a3cb4cb8695 2024-11-07T14:16:34,308 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5518b4545168415c80132a3cb4cb8695, entries=150, sequenceid=260, filesize=30.5 K 2024-11-07T14:16:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/916b86b22ffd46f495f8d4fdf5be08c0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/916b86b22ffd46f495f8d4fdf5be08c0 2024-11-07T14:16:34,320 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/916b86b22ffd46f495f8d4fdf5be08c0, entries=150, sequenceid=260, filesize=12.0 K 2024-11-07T14:16:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/6aa7a99eefb648709d0dbf6c85a9ffcb as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/6aa7a99eefb648709d0dbf6c85a9ffcb 2024-11-07T14:16:34,333 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/6aa7a99eefb648709d0dbf6c85a9ffcb, entries=150, sequenceid=260, filesize=12.0 K 2024-11-07T14:16:34,333 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 57f705175086737d129131015f71452d in 1831ms, sequenceid=260, compaction requested=true 2024-11-07T14:16:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-07T14:16:34,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-07T14:16:34,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-07T14:16:34,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9920 sec 2024-11-07T14:16:34,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.9960 sec 2024-11-07T14:16:34,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:34,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:16:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:34,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fe5025c06e4849c8ac62c3ee684e6856_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988994467/Put/seqid=0 2024-11-07T14:16:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T14:16:34,503 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:34,503 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-07T14:16:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T14:16:34,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:34,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:34,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:34,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742463_1639 (size=14994) 2024-11-07T14:16:34,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989054513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989054519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989054523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,531 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:34,547 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fe5025c06e4849c8ac62c3ee684e6856_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fe5025c06e4849c8ac62c3ee684e6856_57f705175086737d129131015f71452d 2024-11-07T14:16:34,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bcf97637d2ab47ae9568bc81ffe4a408, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:34,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bcf97637d2ab47ae9568bc81ffe4a408 is 175, key is test_row_0/A:col10/1730988994467/Put/seqid=0 2024-11-07T14:16:34,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742464_1640 (size=39949) 2024-11-07T14:16:34,600 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bcf97637d2ab47ae9568bc81ffe4a408 2024-11-07T14:16:34,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/78e3ddb8782440b0b435371c2a7fa3a4 is 50, key is test_row_0/B:col10/1730988994467/Put/seqid=0 2024-11-07T14:16:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T14:16:34,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989054623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989054632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989054633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-07T14:16:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:34,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:34,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742465_1641 (size=12301) 2024-11-07T14:16:34,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-07T14:16:34,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:34,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:34,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:34,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:34,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T14:16:34,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989054832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989054842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989054842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:34,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-07T14:16:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:34,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:34,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:35,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/78e3ddb8782440b0b435371c2a7fa3a4 2024-11-07T14:16:35,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/0e0d76f6937240c0ac6de6a2b2c96128 is 50, key is test_row_0/C:col10/1730988994467/Put/seqid=0 2024-11-07T14:16:35,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-07T14:16:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:35,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:35,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:35,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T14:16:35,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742466_1642 (size=12301) 2024-11-07T14:16:35,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989055146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/0e0d76f6937240c0ac6de6a2b2c96128 2024-11-07T14:16:35,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989055152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989055152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/bcf97637d2ab47ae9568bc81ffe4a408 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bcf97637d2ab47ae9568bc81ffe4a408 2024-11-07T14:16:35,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bcf97637d2ab47ae9568bc81ffe4a408, entries=200, sequenceid=278, filesize=39.0 K 2024-11-07T14:16:35,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/78e3ddb8782440b0b435371c2a7fa3a4 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/78e3ddb8782440b0b435371c2a7fa3a4 2024-11-07T14:16:35,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/78e3ddb8782440b0b435371c2a7fa3a4, entries=150, sequenceid=278, filesize=12.0 K 2024-11-07T14:16:35,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/0e0d76f6937240c0ac6de6a2b2c96128 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/0e0d76f6937240c0ac6de6a2b2c96128 2024-11-07T14:16:35,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/0e0d76f6937240c0ac6de6a2b2c96128, entries=150, sequenceid=278, filesize=12.0 K 2024-11-07T14:16:35,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 57f705175086737d129131015f71452d in 752ms, sequenceid=278, compaction requested=true 2024-11-07T14:16:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:35,221 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:16:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:35,222 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:16:35,222 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133926 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:16:35,222 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/A is initiating minor compaction (all files) 2024-11-07T14:16:35,223 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/A in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:35,223 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/faa86b443bd44993ae6add181c5b08da, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cade969da0454ca49449eb2edb1a906d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5518b4545168415c80132a3cb4cb8695, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bcf97637d2ab47ae9568bc81ffe4a408] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=130.8 K 2024-11-07T14:16:35,223 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:35,223 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/faa86b443bd44993ae6add181c5b08da, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cade969da0454ca49449eb2edb1a906d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5518b4545168415c80132a3cb4cb8695, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bcf97637d2ab47ae9568bc81ffe4a408] 2024-11-07T14:16:35,223 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa86b443bd44993ae6add181c5b08da, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1730988988772 2024-11-07T14:16:35,224 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting cade969da0454ca49449eb2edb1a906d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1730988989954 2024-11-07T14:16:35,224 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5518b4545168415c80132a3cb4cb8695, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1730988991139 2024-11-07T14:16:35,224 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:16:35,224 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/B is initiating minor compaction (all files) 2024-11-07T14:16:35,224 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/B in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:35,224 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1ecc2f2172f149b5bdb694a32535a7d3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e8829ced69ba4be7a9d133d28c6f1f30, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/916b86b22ffd46f495f8d4fdf5be08c0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/78e3ddb8782440b0b435371c2a7fa3a4] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=48.3 K 2024-11-07T14:16:35,225 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcf97637d2ab47ae9568bc81ffe4a408, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730988993301 2024-11-07T14:16:35,225 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ecc2f2172f149b5bdb694a32535a7d3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1730988988772 2024-11-07T14:16:35,225 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting e8829ced69ba4be7a9d133d28c6f1f30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1730988989954 2024-11-07T14:16:35,226 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 916b86b22ffd46f495f8d4fdf5be08c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1730988991139 2024-11-07T14:16:35,226 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e3ddb8782440b0b435371c2a7fa3a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730988993301 2024-11-07T14:16:35,239 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:35,243 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#B#compaction#548 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:35,244 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/a1af10b3f9cf41ee98db4ca85278d39e is 50, key is test_row_0/B:col10/1730988994467/Put/seqid=0 2024-11-07T14:16:35,247 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107ef97ec3d190e47438ed55a1c4bba3b34_57f705175086737d129131015f71452d store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:35,250 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107ef97ec3d190e47438ed55a1c4bba3b34_57f705175086737d129131015f71452d, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:35,250 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ef97ec3d190e47438ed55a1c4bba3b34_57f705175086737d129131015f71452d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:35,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-07T14:16:35,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:35,284 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T14:16:35,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742467_1643 (size=12949) 2024-11-07T14:16:35,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742468_1644 (size=4469) 2024-11-07T14:16:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110778b8cdb3b8294e9eb1fcd561aae10f49_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988994492/Put/seqid=0 2024-11-07T14:16:35,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742469_1645 (size=12454) 2024-11-07T14:16:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:35,333 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110778b8cdb3b8294e9eb1fcd561aae10f49_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110778b8cdb3b8294e9eb1fcd561aae10f49_57f705175086737d129131015f71452d 2024-11-07T14:16:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/aca8f294a8a64c74a8c5c1db94b625a7, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/aca8f294a8a64c74a8c5c1db94b625a7 is 175, key is test_row_0/A:col10/1730988994492/Put/seqid=0 2024-11-07T14:16:35,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742470_1646 (size=31255) 2024-11-07T14:16:35,381 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/aca8f294a8a64c74a8c5c1db94b625a7 2024-11-07T14:16:35,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/98372da0481a43a1946ca032512da3a9 is 50, key is test_row_0/B:col10/1730988994492/Put/seqid=0 2024-11-07T14:16:35,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742471_1647 (size=12301) 2024-11-07T14:16:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T14:16:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:35,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:35,688 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#A#compaction#547 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:35,688 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/0f4ca40113ee4ab3a5007958dabf2fd6 is 175, key is test_row_0/A:col10/1730988994467/Put/seqid=0 2024-11-07T14:16:35,693 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/a1af10b3f9cf41ee98db4ca85278d39e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a1af10b3f9cf41ee98db4ca85278d39e 2024-11-07T14:16:35,698 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57f705175086737d129131015f71452d/B of 57f705175086737d129131015f71452d into a1af10b3f9cf41ee98db4ca85278d39e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:35,698 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:35,698 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/B, priority=12, startTime=1730988995221; duration=0sec 2024-11-07T14:16:35,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742472_1648 (size=31903) 2024-11-07T14:16:35,698 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:35,698 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:B 2024-11-07T14:16:35,698 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T14:16:35,699 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T14:16:35,699 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/C is initiating minor compaction (all files) 2024-11-07T14:16:35,699 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/C in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:35,700 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/63f84570003d458998a0b1abb9dd2282, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b03e55fa8288446286bb67f2a87e8e5f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/6aa7a99eefb648709d0dbf6c85a9ffcb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/0e0d76f6937240c0ac6de6a2b2c96128] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=48.3 K 2024-11-07T14:16:35,700 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f84570003d458998a0b1abb9dd2282, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1730988988772 2024-11-07T14:16:35,700 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting b03e55fa8288446286bb67f2a87e8e5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1730988989954 2024-11-07T14:16:35,700 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa7a99eefb648709d0dbf6c85a9ffcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1730988991139 2024-11-07T14:16:35,700 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e0d76f6937240c0ac6de6a2b2c96128, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730988993301 2024-11-07T14:16:35,702 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/0f4ca40113ee4ab3a5007958dabf2fd6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/0f4ca40113ee4ab3a5007958dabf2fd6 2024-11-07T14:16:35,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989055699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989055699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989055699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,706 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57f705175086737d129131015f71452d/A of 57f705175086737d129131015f71452d into 0f4ca40113ee4ab3a5007958dabf2fd6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:35,707 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:35,707 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/A, priority=12, startTime=1730988995221; duration=0sec 2024-11-07T14:16:35,707 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:35,707 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:A 2024-11-07T14:16:35,711 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#C#compaction#551 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:35,711 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/70c48f3bf8864446b48c58bfcfd274e6 is 50, key is test_row_0/C:col10/1730988994467/Put/seqid=0 2024-11-07T14:16:35,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742473_1649 (size=12949) 2024-11-07T14:16:35,743 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/70c48f3bf8864446b48c58bfcfd274e6 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/70c48f3bf8864446b48c58bfcfd274e6 2024-11-07T14:16:35,751 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57f705175086737d129131015f71452d/C of 57f705175086737d129131015f71452d into 70c48f3bf8864446b48c58bfcfd274e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:35,751 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:35,751 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/C, priority=12, startTime=1730988995221; duration=0sec 2024-11-07T14:16:35,751 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:35,751 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:C 2024-11-07T14:16:35,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989055812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989055812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989055812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:35,828 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/98372da0481a43a1946ca032512da3a9 2024-11-07T14:16:35,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/140d80ac520840149a98f8ed007d675f is 50, key is test_row_0/C:col10/1730988994492/Put/seqid=0 2024-11-07T14:16:35,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742474_1650 (size=12301) 2024-11-07T14:16:36,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989056022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989056022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989056022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,247 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/140d80ac520840149a98f8ed007d675f 2024-11-07T14:16:36,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/aca8f294a8a64c74a8c5c1db94b625a7 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/aca8f294a8a64c74a8c5c1db94b625a7 2024-11-07T14:16:36,279 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/aca8f294a8a64c74a8c5c1db94b625a7, entries=150, sequenceid=296, filesize=30.5 K 2024-11-07T14:16:36,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/98372da0481a43a1946ca032512da3a9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/98372da0481a43a1946ca032512da3a9 2024-11-07T14:16:36,308 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/98372da0481a43a1946ca032512da3a9, entries=150, sequenceid=296, filesize=12.0 K 2024-11-07T14:16:36,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/140d80ac520840149a98f8ed007d675f as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/140d80ac520840149a98f8ed007d675f 2024-11-07T14:16:36,330 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/140d80ac520840149a98f8ed007d675f, entries=150, sequenceid=296, filesize=12.0 K 2024-11-07T14:16:36,331 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=114.05 KB/116790 for 57f705175086737d129131015f71452d in 1047ms, sequenceid=296, compaction requested=false 2024-11-07T14:16:36,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:36,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:36,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-07T14:16:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-07T14:16:36,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-07T14:16:36,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8270 sec 2024-11-07T14:16:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T14:16:36,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T14:16:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:36,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.8330 sec 2024-11-07T14:16:36,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107011dbcd11c2241a5908c954e9dff41e4_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988996335/Put/seqid=0 2024-11-07T14:16:36,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742475_1651 (size=17534) 2024-11-07T14:16:36,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989056358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989056364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989056364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989056473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989056473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989056473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-07T14:16:36,663 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:36,667 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-07T14:16:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-07T14:16:36,669 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T14:16:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T14:16:36,669 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T14:16:36,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T14:16:36,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989056682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989056682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989056683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34946 deadline: 1730989056712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,714 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_1' in 57f705175086737d129131015f71452d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,753 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:36,759 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107011dbcd11c2241a5908c954e9dff41e4_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107011dbcd11c2241a5908c954e9dff41e4_57f705175086737d129131015f71452d 2024-11-07T14:16:36,763 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/98b8aac6a3ec4ada9a173422f272ed60, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:36,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/98b8aac6a3ec4ada9a173422f272ed60 is 175, key is test_row_0/A:col10/1730988996335/Put/seqid=0 2024-11-07T14:16:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T14:16:36,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1730989056772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,774 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b7cc9d764de6,37737,1730988826985 for 'test_row_0' in 57f705175086737d129131015f71452d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T14:16:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742476_1652 (size=48639) 2024-11-07T14:16:36,776 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/98b8aac6a3ec4ada9a173422f272ed60 2024-11-07T14:16:36,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/3e0fa928e94c4eebb188a967bbbbae4c is 50, key is test_row_0/B:col10/1730988996335/Put/seqid=0 2024-11-07T14:16:36,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742477_1653 (size=12301) 2024-11-07T14:16:36,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-07T14:16:36,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:36,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:36,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:36,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:36,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:36,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-07T14:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:36,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T14:16:36,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989056992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989056992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:36,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989056995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:37,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-07T14:16:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/3e0fa928e94c4eebb188a967bbbbae4c 2024-11-07T14:16:37,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9df823be2e714507b454cf81590c656a is 50, key is test_row_0/C:col10/1730988996335/Put/seqid=0 2024-11-07T14:16:37,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742478_1654 (size=12301) 2024-11-07T14:16:37,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-07T14:16:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T14:16:37,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-07T14:16:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35002 deadline: 1730989057502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:37,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34944 deadline: 1730989057502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:37,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T14:16:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37737 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34990 deadline: 1730989057513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 , the old value is region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T14:16:37,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., hostname=b7cc9d764de6,37737,1730988826985, seqNum=5 because the exception is null or not the one we care about 2024-11-07T14:16:37,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-07T14:16:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T14:16:37,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9df823be2e714507b454cf81590c656a 2024-11-07T14:16:37,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/98b8aac6a3ec4ada9a173422f272ed60 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/98b8aac6a3ec4ada9a173422f272ed60 2024-11-07T14:16:37,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/98b8aac6a3ec4ada9a173422f272ed60, entries=250, sequenceid=322, filesize=47.5 K 2024-11-07T14:16:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/3e0fa928e94c4eebb188a967bbbbae4c as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/3e0fa928e94c4eebb188a967bbbbae4c 2024-11-07T14:16:37,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/3e0fa928e94c4eebb188a967bbbbae4c, entries=150, sequenceid=322, filesize=12.0 K 2024-11-07T14:16:37,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9df823be2e714507b454cf81590c656a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9df823be2e714507b454cf81590c656a 2024-11-07T14:16:37,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9df823be2e714507b454cf81590c656a, entries=150, sequenceid=322, filesize=12.0 K 2024-11-07T14:16:37,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 57f705175086737d129131015f71452d in 1359ms, sequenceid=322, compaction requested=true 2024-11-07T14:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T14:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T14:16:37,695 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57f705175086737d129131015f71452d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T14:16:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T14:16:37,696 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:37,703 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:37,703 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/A is initiating minor compaction (all files) 2024-11-07T14:16:37,703 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/A in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,703 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/0f4ca40113ee4ab3a5007958dabf2fd6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/aca8f294a8a64c74a8c5c1db94b625a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/98b8aac6a3ec4ada9a173422f272ed60] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=109.2 K 2024-11-07T14:16:37,703 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,703 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. files: [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/0f4ca40113ee4ab3a5007958dabf2fd6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/aca8f294a8a64c74a8c5c1db94b625a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/98b8aac6a3ec4ada9a173422f272ed60] 2024-11-07T14:16:37,707 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f4ca40113ee4ab3a5007958dabf2fd6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730988993301 2024-11-07T14:16:37,711 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:37,711 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting aca8f294a8a64c74a8c5c1db94b625a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730988994492 2024-11-07T14:16:37,711 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/B is initiating minor compaction (all files) 2024-11-07T14:16:37,711 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/B in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,711 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a1af10b3f9cf41ee98db4ca85278d39e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/98372da0481a43a1946ca032512da3a9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/3e0fa928e94c4eebb188a967bbbbae4c] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=36.7 K 2024-11-07T14:16:37,711 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98b8aac6a3ec4ada9a173422f272ed60, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730988995697 2024-11-07T14:16:37,711 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting a1af10b3f9cf41ee98db4ca85278d39e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730988993301 2024-11-07T14:16:37,712 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 98372da0481a43a1946ca032512da3a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730988994492 2024-11-07T14:16:37,712 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e0fa928e94c4eebb188a967bbbbae4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730988995698 2024-11-07T14:16:37,727 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:37,732 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#B#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:37,732 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/060e1d83251f40c28302662c551ce20e is 50, key is test_row_0/B:col10/1730988996335/Put/seqid=0 2024-11-07T14:16:37,739 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107c90c0a641303463aab4f12f1d6e8b1bd_57f705175086737d129131015f71452d store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:37,741 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107c90c0a641303463aab4f12f1d6e8b1bd_57f705175086737d129131015f71452d, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:37,741 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c90c0a641303463aab4f12f1d6e8b1bd_57f705175086737d129131015f71452d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:37,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37737 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-07T14:16:37,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:37,750 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T14:16:37,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742479_1655 (size=13051) 2024-11-07T14:16:37,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742480_1656 (size=4469) 2024-11-07T14:16:37,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073cd417edb9864d998525a27444e0ad85_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730988996355/Put/seqid=0 2024-11-07T14:16:37,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742481_1657 (size=12454) 2024-11-07T14:16:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T14:16:38,163 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#A#compaction#556 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:38,164 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/8264b0e716ca40628632ce60af9e38b3 is 175, key is test_row_0/A:col10/1730988996335/Put/seqid=0 2024-11-07T14:16:38,173 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/060e1d83251f40c28302662c551ce20e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/060e1d83251f40c28302662c551ce20e 2024-11-07T14:16:38,185 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/B of 57f705175086737d129131015f71452d into 060e1d83251f40c28302662c551ce20e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:38,185 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:38,185 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/B, priority=13, startTime=1730988997695; duration=0sec 2024-11-07T14:16:38,186 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T14:16:38,186 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:B 2024-11-07T14:16:38,186 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T14:16:38,187 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T14:16:38,188 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1541): 57f705175086737d129131015f71452d/C is initiating minor compaction (all files) 2024-11-07T14:16:38,188 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57f705175086737d129131015f71452d/C in TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:38,188 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/70c48f3bf8864446b48c58bfcfd274e6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/140d80ac520840149a98f8ed007d675f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9df823be2e714507b454cf81590c656a] into tmpdir=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp, totalSize=36.7 K 2024-11-07T14:16:38,189 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c48f3bf8864446b48c58bfcfd274e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730988993301 2024-11-07T14:16:38,190 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 140d80ac520840149a98f8ed007d675f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1730988994492 2024-11-07T14:16:38,190 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df823be2e714507b454cf81590c656a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1730988995698 2024-11-07T14:16:38,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:38,198 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073cd417edb9864d998525a27444e0ad85_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073cd417edb9864d998525a27444e0ad85_57f705175086737d129131015f71452d 2024-11-07T14:16:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/47de55aad3f24576ae1f4255ad7ea01b, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/47de55aad3f24576ae1f4255ad7ea01b is 175, key is test_row_0/A:col10/1730988996355/Put/seqid=0 2024-11-07T14:16:38,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742482_1658 (size=32005) 2024-11-07T14:16:38,205 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57f705175086737d129131015f71452d#C#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T14:16:38,205 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/cc6e189006744ed58b15746c176c4f6a is 50, key is test_row_0/C:col10/1730988996335/Put/seqid=0 2024-11-07T14:16:38,209 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/8264b0e716ca40628632ce60af9e38b3 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/8264b0e716ca40628632ce60af9e38b3 2024-11-07T14:16:38,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742483_1659 (size=31255) 2024-11-07T14:16:38,212 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/47de55aad3f24576ae1f4255ad7ea01b 2024-11-07T14:16:38,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742484_1660 (size=13051) 2024-11-07T14:16:38,216 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/A of 57f705175086737d129131015f71452d into 8264b0e716ca40628632ce60af9e38b3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:38,216 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:38,216 INFO [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/A, priority=13, startTime=1730988997695; duration=0sec 2024-11-07T14:16:38,216 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:38,216 DEBUG [RS:0;b7cc9d764de6:37737-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:A 2024-11-07T14:16:38,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e1a05270a6fd4288befa3cf94f21cafe is 50, key is test_row_0/B:col10/1730988996355/Put/seqid=0 2024-11-07T14:16:38,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742485_1661 (size=12301) 2024-11-07T14:16:38,437 INFO [Thread-2527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2527. 2024-11-07T14:16:38,437 INFO [Thread-2521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2521. 2024-11-07T14:16:38,437 DEBUG [Thread-2521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:38,437 DEBUG [Thread-2527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:38,437 DEBUG [Thread-2521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,437 DEBUG [Thread-2527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,437 DEBUG [Thread-2527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:38,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:38,437 DEBUG [Thread-2521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,438 INFO [Thread-2525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2525. 2024-11-07T14:16:38,438 DEBUG [Thread-2525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:38,438 DEBUG [Thread-2525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,438 DEBUG [Thread-2525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:38,442 INFO [Thread-2529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2529. 2024-11-07T14:16:38,443 DEBUG [Thread-2529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:38,443 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,443 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,443 INFO [Thread-2523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2523. 2024-11-07T14:16:38,443 DEBUG [Thread-2523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:38,443 DEBUG [Thread-2523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,443 DEBUG [Thread-2523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:38,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37737 {}] regionserver.HRegion(8855): Flush requested on 57f705175086737d129131015f71452d 2024-11-07T14:16:38,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. as already flushing 2024-11-07T14:16:38,514 INFO [Thread-2510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2510. 2024-11-07T14:16:38,514 DEBUG [Thread-2510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:38,514 DEBUG [Thread-2510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,514 DEBUG [Thread-2510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,514 INFO [Thread-2518 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2518. 2024-11-07T14:16:38,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:38,514 DEBUG [Thread-2518 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:38,514 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,514 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:38,524 INFO [Thread-2516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2516. 2024-11-07T14:16:38,524 DEBUG [Thread-2516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:38,524 DEBUG [Thread-2516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,524 DEBUG [Thread-2516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:38,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:38,621 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/cc6e189006744ed58b15746c176c4f6a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/cc6e189006744ed58b15746c176c4f6a 2024-11-07T14:16:38,625 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57f705175086737d129131015f71452d/C of 57f705175086737d129131015f71452d into cc6e189006744ed58b15746c176c4f6a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T14:16:38,625 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:38,625 INFO [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., storeName=57f705175086737d129131015f71452d/C, priority=13, startTime=1730988997695; duration=0sec 2024-11-07T14:16:38,625 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T14:16:38,625 DEBUG [RS:0;b7cc9d764de6:37737-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57f705175086737d129131015f71452d:C 2024-11-07T14:16:38,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e1a05270a6fd4288befa3cf94f21cafe 2024-11-07T14:16:38,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9e29c8f491144efe9068aba1c5b2601a is 50, key is test_row_0/C:col10/1730988996355/Put/seqid=0 2024-11-07T14:16:38,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742486_1662 (size=12301) 2024-11-07T14:16:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T14:16:39,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9e29c8f491144efe9068aba1c5b2601a 2024-11-07T14:16:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/47de55aad3f24576ae1f4255ad7ea01b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/47de55aad3f24576ae1f4255ad7ea01b 2024-11-07T14:16:39,066 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/47de55aad3f24576ae1f4255ad7ea01b, entries=150, sequenceid=336, filesize=30.5 K 2024-11-07T14:16:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/e1a05270a6fd4288befa3cf94f21cafe as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e1a05270a6fd4288befa3cf94f21cafe 2024-11-07T14:16:39,070 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e1a05270a6fd4288befa3cf94f21cafe, entries=150, sequenceid=336, filesize=12.0 K 2024-11-07T14:16:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/9e29c8f491144efe9068aba1c5b2601a as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9e29c8f491144efe9068aba1c5b2601a 2024-11-07T14:16:39,074 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9e29c8f491144efe9068aba1c5b2601a, entries=150, sequenceid=336, filesize=12.0 K 2024-11-07T14:16:39,075 INFO [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=20.13 KB/20610 for 57f705175086737d129131015f71452d in 1325ms, sequenceid=336, compaction requested=false 2024-11-07T14:16:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for 57f705175086737d129131015f71452d: 2024-11-07T14:16:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b7cc9d764de6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-07T14:16:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-07T14:16:39,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-07T14:16:39,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4080 sec 2024-11-07T14:16:39,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 2.4130 sec 2024-11-07T14:16:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-07T14:16:40,822 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:45,077 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T14:16:46,803 INFO [Thread-2514 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2514. 2024-11-07T14:16:46,803 DEBUG [Thread-2514 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:46,803 DEBUG [Thread-2514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:46,804 DEBUG [Thread-2514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:46,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:46,845 INFO [Thread-2512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2512. 2024-11-07T14:16:46,845 DEBUG [Thread-2512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T14:16:46,845 DEBUG [Thread-2512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:46,846 DEBUG [Thread-2512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:46,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3463 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3411 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3349 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3399 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3398 2024-11-07T14:16:46,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T14:16:46,846 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T14:16:46,847 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:16:46,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:46,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:46,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:46,848 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-07T14:16:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T14:16:46,853 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989006853"}]},"ts":"1730989006853"} 2024-11-07T14:16:46,855 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T14:16:46,855 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T14:16:46,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T14:16:46,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, UNASSIGN}] 2024-11-07T14:16:46,858 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, UNASSIGN 2024-11-07T14:16:46,859 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=180 updating hbase:meta row=57f705175086737d129131015f71452d, regionState=CLOSING, regionLocation=b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:46,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, UNASSIGN because future has completed 2024-11-07T14:16:46,861 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T14:16:46,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; CloseRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985}] 2024-11-07T14:16:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T14:16:47,014 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(122): Close 57f705175086737d129131015f71452d 2024-11-07T14:16:47,014 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T14:16:47,014 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1722): Closing 57f705175086737d129131015f71452d, disabling compactions & flushes 2024-11-07T14:16:47,014 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:47,014 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:47,014 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. after waiting 0 ms 2024-11-07T14:16:47,014 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:47,015 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(2902): Flushing 57f705175086737d129131015f71452d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T14:16:47,024 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079be6e22a1f5042a68793ec3d55ae330b_57f705175086737d129131015f71452d is 50, key is test_row_0/A:col10/1730989006843/Put/seqid=0 2024-11-07T14:16:47,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742487_1663 (size=12454) 2024-11-07T14:16:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T14:16:47,428 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T14:16:47,432 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079be6e22a1f5042a68793ec3d55ae330b_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079be6e22a1f5042a68793ec3d55ae330b_57f705175086737d129131015f71452d 2024-11-07T14:16:47,433 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/111d251d97f54783bfd9a75318b839d9, store: [table=TestAcidGuarantees family=A region=57f705175086737d129131015f71452d] 2024-11-07T14:16:47,433 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/111d251d97f54783bfd9a75318b839d9 is 175, key is test_row_0/A:col10/1730989006843/Put/seqid=0 2024-11-07T14:16:47,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742488_1664 (size=31255) 2024-11-07T14:16:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T14:16:47,837 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/111d251d97f54783bfd9a75318b839d9 2024-11-07T14:16:47,842 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/a44a6189e7664394af647945048160a0 is 50, key is test_row_0/B:col10/1730989006843/Put/seqid=0 2024-11-07T14:16:47,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742489_1665 (size=12301) 2024-11-07T14:16:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T14:16:48,246 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/a44a6189e7664394af647945048160a0 2024-11-07T14:16:48,255 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b13e8ab4386a43348cc1ce3e12e4cb31 is 50, key is test_row_0/C:col10/1730989006843/Put/seqid=0 2024-11-07T14:16:48,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742490_1666 (size=12301) 2024-11-07T14:16:48,660 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b13e8ab4386a43348cc1ce3e12e4cb31 2024-11-07T14:16:48,665 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/A/111d251d97f54783bfd9a75318b839d9 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/111d251d97f54783bfd9a75318b839d9 2024-11-07T14:16:48,668 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/111d251d97f54783bfd9a75318b839d9, entries=150, sequenceid=347, filesize=30.5 K 2024-11-07T14:16:48,668 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/B/a44a6189e7664394af647945048160a0 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a44a6189e7664394af647945048160a0 2024-11-07T14:16:48,671 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a44a6189e7664394af647945048160a0, entries=150, sequenceid=347, filesize=12.0 K 2024-11-07T14:16:48,672 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/.tmp/C/b13e8ab4386a43348cc1ce3e12e4cb31 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b13e8ab4386a43348cc1ce3e12e4cb31 2024-11-07T14:16:48,675 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b13e8ab4386a43348cc1ce3e12e4cb31, entries=150, sequenceid=347, filesize=12.0 K 2024-11-07T14:16:48,675 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 57f705175086737d129131015f71452d in 1661ms, sequenceid=347, compaction requested=true 2024-11-07T14:16:48,676 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/a41d2251585c4aa1ba437a19852a3d65, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/2c6d4378b3e346279e7067a9019ee33b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/272686f37fd64d1c82aa367ecb2de5b2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/58c9cb18aeb240c4aabdbdba509beece, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd32b067be0c48e4bfcfa4fe806b4660, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c883e037dbd24fd69d54f61a20f0e1c2, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd4ce2a1fc284d04a32c0aae564c573d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/720123dd9ed2489d864266acb17d71bd, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c06d602bae17447283f24dcb65055c1e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/89f15837114b49f68107d2ef7fc96d25, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/17b98ab3f47e4f89b0a51cedfae33f29, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/efc35a984ad3452a92f0fddf104b40b9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5239b862a8cb4ea9ad418e4614f19937, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cb2a61111cf946208205ea8b49978dd6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/9fa35368165d4c478f8fbacdfaeebc70, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/faa86b443bd44993ae6add181c5b08da, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cade969da0454ca49449eb2edb1a906d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5518b4545168415c80132a3cb4cb8695, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bcf97637d2ab47ae9568bc81ffe4a408, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/0f4ca40113ee4ab3a5007958dabf2fd6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/aca8f294a8a64c74a8c5c1db94b625a7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/98b8aac6a3ec4ada9a173422f272ed60] to archive 2024-11-07T14:16:48,676 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:16:48,677 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/a41d2251585c4aa1ba437a19852a3d65 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/a41d2251585c4aa1ba437a19852a3d65 2024-11-07T14:16:48,678 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/2c6d4378b3e346279e7067a9019ee33b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/2c6d4378b3e346279e7067a9019ee33b 2024-11-07T14:16:48,679 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/272686f37fd64d1c82aa367ecb2de5b2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/272686f37fd64d1c82aa367ecb2de5b2 2024-11-07T14:16:48,680 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/58c9cb18aeb240c4aabdbdba509beece to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/58c9cb18aeb240c4aabdbdba509beece 2024-11-07T14:16:48,681 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd32b067be0c48e4bfcfa4fe806b4660 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd32b067be0c48e4bfcfa4fe806b4660 2024-11-07T14:16:48,682 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c883e037dbd24fd69d54f61a20f0e1c2 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c883e037dbd24fd69d54f61a20f0e1c2 2024-11-07T14:16:48,683 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd4ce2a1fc284d04a32c0aae564c573d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bd4ce2a1fc284d04a32c0aae564c573d 2024-11-07T14:16:48,683 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/720123dd9ed2489d864266acb17d71bd to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/720123dd9ed2489d864266acb17d71bd 2024-11-07T14:16:48,684 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c06d602bae17447283f24dcb65055c1e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/c06d602bae17447283f24dcb65055c1e 2024-11-07T14:16:48,685 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/89f15837114b49f68107d2ef7fc96d25 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/89f15837114b49f68107d2ef7fc96d25 2024-11-07T14:16:48,685 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/17b98ab3f47e4f89b0a51cedfae33f29 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/17b98ab3f47e4f89b0a51cedfae33f29 2024-11-07T14:16:48,686 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/efc35a984ad3452a92f0fddf104b40b9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/efc35a984ad3452a92f0fddf104b40b9 2024-11-07T14:16:48,687 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5239b862a8cb4ea9ad418e4614f19937 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5239b862a8cb4ea9ad418e4614f19937 2024-11-07T14:16:48,688 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cb2a61111cf946208205ea8b49978dd6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cb2a61111cf946208205ea8b49978dd6 2024-11-07T14:16:48,688 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/9fa35368165d4c478f8fbacdfaeebc70 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/9fa35368165d4c478f8fbacdfaeebc70 2024-11-07T14:16:48,689 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/faa86b443bd44993ae6add181c5b08da to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/faa86b443bd44993ae6add181c5b08da 2024-11-07T14:16:48,689 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cade969da0454ca49449eb2edb1a906d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/cade969da0454ca49449eb2edb1a906d 2024-11-07T14:16:48,690 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5518b4545168415c80132a3cb4cb8695 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/5518b4545168415c80132a3cb4cb8695 2024-11-07T14:16:48,691 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bcf97637d2ab47ae9568bc81ffe4a408 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/bcf97637d2ab47ae9568bc81ffe4a408 2024-11-07T14:16:48,691 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/0f4ca40113ee4ab3a5007958dabf2fd6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/0f4ca40113ee4ab3a5007958dabf2fd6 2024-11-07T14:16:48,692 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/aca8f294a8a64c74a8c5c1db94b625a7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/aca8f294a8a64c74a8c5c1db94b625a7 2024-11-07T14:16:48,692 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/98b8aac6a3ec4ada9a173422f272ed60 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/98b8aac6a3ec4ada9a173422f272ed60 2024-11-07T14:16:48,693 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b737df9c01fc498989acea9710b69f4d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/25b2a7bac3c048889f3ffdfe33f4c821, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b3d11906f98d414b98064ca6f9194867, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1cd92882b30a438dbf83ebc0e2ae723f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/067fd40b46d44dbda5e4b7f28f662c2e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c90986a35a7147b9a07960433fa4be4b, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c771a64ac53441cb811db91501dd7c22, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/6fa8cc8efdb441a3956d26fe66816be8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/12cf4a576e504243807e217103383f67, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/83bfa0f5317e41649c46f91e462db2a6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/08b1fb0ba160493390738f9effecb639, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/f7401e5812314ac7980f69df7f213137, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/583d8ef250224f86a9a02631df7b4df7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/016920d959d44012ab57eb45c60b9e9d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1ecc2f2172f149b5bdb694a32535a7d3, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e0aba444fa444cd8a64e068fd113f264, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e8829ced69ba4be7a9d133d28c6f1f30, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/916b86b22ffd46f495f8d4fdf5be08c0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a1af10b3f9cf41ee98db4ca85278d39e, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/78e3ddb8782440b0b435371c2a7fa3a4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/98372da0481a43a1946ca032512da3a9, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/3e0fa928e94c4eebb188a967bbbbae4c] to archive 2024-11-07T14:16:48,694 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:16:48,695 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b737df9c01fc498989acea9710b69f4d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b737df9c01fc498989acea9710b69f4d 2024-11-07T14:16:48,695 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/25b2a7bac3c048889f3ffdfe33f4c821 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/25b2a7bac3c048889f3ffdfe33f4c821 2024-11-07T14:16:48,696 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b3d11906f98d414b98064ca6f9194867 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/b3d11906f98d414b98064ca6f9194867 2024-11-07T14:16:48,697 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1cd92882b30a438dbf83ebc0e2ae723f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1cd92882b30a438dbf83ebc0e2ae723f 2024-11-07T14:16:48,697 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/067fd40b46d44dbda5e4b7f28f662c2e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/067fd40b46d44dbda5e4b7f28f662c2e 2024-11-07T14:16:48,698 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c90986a35a7147b9a07960433fa4be4b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c90986a35a7147b9a07960433fa4be4b 2024-11-07T14:16:48,699 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c771a64ac53441cb811db91501dd7c22 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/c771a64ac53441cb811db91501dd7c22 2024-11-07T14:16:48,700 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/6fa8cc8efdb441a3956d26fe66816be8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/6fa8cc8efdb441a3956d26fe66816be8 2024-11-07T14:16:48,700 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/12cf4a576e504243807e217103383f67 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/12cf4a576e504243807e217103383f67 2024-11-07T14:16:48,701 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/83bfa0f5317e41649c46f91e462db2a6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/83bfa0f5317e41649c46f91e462db2a6 2024-11-07T14:16:48,702 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/08b1fb0ba160493390738f9effecb639 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/08b1fb0ba160493390738f9effecb639 2024-11-07T14:16:48,703 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/f7401e5812314ac7980f69df7f213137 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/f7401e5812314ac7980f69df7f213137 2024-11-07T14:16:48,703 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/583d8ef250224f86a9a02631df7b4df7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/583d8ef250224f86a9a02631df7b4df7 2024-11-07T14:16:48,704 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/016920d959d44012ab57eb45c60b9e9d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/016920d959d44012ab57eb45c60b9e9d 2024-11-07T14:16:48,705 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1ecc2f2172f149b5bdb694a32535a7d3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/1ecc2f2172f149b5bdb694a32535a7d3 2024-11-07T14:16:48,705 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e0aba444fa444cd8a64e068fd113f264 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e0aba444fa444cd8a64e068fd113f264 2024-11-07T14:16:48,706 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e8829ced69ba4be7a9d133d28c6f1f30 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e8829ced69ba4be7a9d133d28c6f1f30 2024-11-07T14:16:48,707 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/916b86b22ffd46f495f8d4fdf5be08c0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/916b86b22ffd46f495f8d4fdf5be08c0 2024-11-07T14:16:48,707 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a1af10b3f9cf41ee98db4ca85278d39e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a1af10b3f9cf41ee98db4ca85278d39e 2024-11-07T14:16:48,708 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/78e3ddb8782440b0b435371c2a7fa3a4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/78e3ddb8782440b0b435371c2a7fa3a4 2024-11-07T14:16:48,709 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/98372da0481a43a1946ca032512da3a9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/98372da0481a43a1946ca032512da3a9 2024-11-07T14:16:48,709 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/3e0fa928e94c4eebb188a967bbbbae4c to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/3e0fa928e94c4eebb188a967bbbbae4c 2024-11-07T14:16:48,711 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/476b21b1164442d7b1684bd708779a23, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/8a7325e3ff604e209123d4d5714fbd30, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/42b8052971484f748a2e1452402f5046, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/2613ae80bccb40198c6224a2324542e8, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/1f8eb895faf74451acf1e93bef5ba389, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/46bbec01bbba4a1f814df5d6fd0616ce, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/677c42bfa5394c22a916fe7e22884b11, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9a3b76a5f6ed43d49194d8d902a9a67d, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/09e7ed99fba04340a7beda8200cead0a, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b9880f070d044730a006f75b275c8dd7, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/de9a0daaceaa4c4fa9982457075e173f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/113b3f1507ef43d2861a24901e54fa31, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/3a8b8b59816347aabb7ddaba04de9dd4, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/f714788da63541d2a67acacf062173a6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/63f84570003d458998a0b1abb9dd2282, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/7e22d0c5d4264f6ba197f6f85f415ba0, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b03e55fa8288446286bb67f2a87e8e5f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/6aa7a99eefb648709d0dbf6c85a9ffcb, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/70c48f3bf8864446b48c58bfcfd274e6, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/0e0d76f6937240c0ac6de6a2b2c96128, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/140d80ac520840149a98f8ed007d675f, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9df823be2e714507b454cf81590c656a] to archive 2024-11-07T14:16:48,711 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T14:16:48,712 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/476b21b1164442d7b1684bd708779a23 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/476b21b1164442d7b1684bd708779a23 2024-11-07T14:16:48,713 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/8a7325e3ff604e209123d4d5714fbd30 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/8a7325e3ff604e209123d4d5714fbd30 2024-11-07T14:16:48,714 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/42b8052971484f748a2e1452402f5046 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/42b8052971484f748a2e1452402f5046 2024-11-07T14:16:48,715 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/2613ae80bccb40198c6224a2324542e8 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/2613ae80bccb40198c6224a2324542e8 2024-11-07T14:16:48,715 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/1f8eb895faf74451acf1e93bef5ba389 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/1f8eb895faf74451acf1e93bef5ba389 2024-11-07T14:16:48,716 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/46bbec01bbba4a1f814df5d6fd0616ce to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/46bbec01bbba4a1f814df5d6fd0616ce 2024-11-07T14:16:48,717 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/677c42bfa5394c22a916fe7e22884b11 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/677c42bfa5394c22a916fe7e22884b11 2024-11-07T14:16:48,718 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9a3b76a5f6ed43d49194d8d902a9a67d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9a3b76a5f6ed43d49194d8d902a9a67d 2024-11-07T14:16:48,718 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/09e7ed99fba04340a7beda8200cead0a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/09e7ed99fba04340a7beda8200cead0a 2024-11-07T14:16:48,719 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b9880f070d044730a006f75b275c8dd7 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b9880f070d044730a006f75b275c8dd7 2024-11-07T14:16:48,720 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/de9a0daaceaa4c4fa9982457075e173f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/de9a0daaceaa4c4fa9982457075e173f 2024-11-07T14:16:48,721 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/113b3f1507ef43d2861a24901e54fa31 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/113b3f1507ef43d2861a24901e54fa31 2024-11-07T14:16:48,721 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/3a8b8b59816347aabb7ddaba04de9dd4 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/3a8b8b59816347aabb7ddaba04de9dd4 2024-11-07T14:16:48,722 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/f714788da63541d2a67acacf062173a6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/f714788da63541d2a67acacf062173a6 2024-11-07T14:16:48,723 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/63f84570003d458998a0b1abb9dd2282 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/63f84570003d458998a0b1abb9dd2282 2024-11-07T14:16:48,724 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/7e22d0c5d4264f6ba197f6f85f415ba0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/7e22d0c5d4264f6ba197f6f85f415ba0 2024-11-07T14:16:48,726 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b03e55fa8288446286bb67f2a87e8e5f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b03e55fa8288446286bb67f2a87e8e5f 2024-11-07T14:16:48,727 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/6aa7a99eefb648709d0dbf6c85a9ffcb to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/6aa7a99eefb648709d0dbf6c85a9ffcb 2024-11-07T14:16:48,727 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/70c48f3bf8864446b48c58bfcfd274e6 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/70c48f3bf8864446b48c58bfcfd274e6 2024-11-07T14:16:48,728 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/0e0d76f6937240c0ac6de6a2b2c96128 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/0e0d76f6937240c0ac6de6a2b2c96128 2024-11-07T14:16:48,729 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/140d80ac520840149a98f8ed007d675f to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/140d80ac520840149a98f8ed007d675f 2024-11-07T14:16:48,730 DEBUG [StoreCloser-TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9df823be2e714507b454cf81590c656a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9df823be2e714507b454cf81590c656a 2024-11-07T14:16:48,734 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/recovered.edits/350.seqid, newMaxSeqId=350, maxSeqId=4 2024-11-07T14:16:48,734 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. 2024-11-07T14:16:48,734 DEBUG [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1676): Region close journal for 57f705175086737d129131015f71452d: Waiting for close lock at 1730989007014Running coprocessor pre-close hooks at 1730989007014Disabling compacts and flushes for region at 1730989007014Disabling writes for close at 1730989007014Obtaining lock to block concurrent updates at 1730989007015 (+1 ms)Preparing flush snapshotting stores in 57f705175086737d129131015f71452d at 1730989007015Finished memstore snapshotting TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730989007015Flushing stores of TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d. at 1730989007016 (+1 ms)Flushing 57f705175086737d129131015f71452d/A: creating writer at 1730989007016Flushing mob file 57f705175086737d129131015f71452d/A: appending metadata at 1730989007024 (+8 ms)Flushing mob file 57f705175086737d129131015f71452d/A: closing flushed file at 1730989007024Flushing 57f705175086737d129131015f71452d/A: appending metadata at 1730989007433 (+409 ms)Flushing 57f705175086737d129131015f71452d/A: closing flushed file at 1730989007433Flushing 57f705175086737d129131015f71452d/B: creating writer at 1730989007840 (+407 ms)Flushing 57f705175086737d129131015f71452d/B: appending metadata at 1730989007842 (+2 ms)Flushing 57f705175086737d129131015f71452d/B: closing flushed file at 1730989007842Flushing 57f705175086737d129131015f71452d/C: creating writer at 1730989008251 (+409 ms)Flushing 57f705175086737d129131015f71452d/C: appending metadata at 1730989008254 (+3 ms)Flushing 57f705175086737d129131015f71452d/C: closing flushed file at 1730989008254Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2098f41c: reopening flushed file at 1730989008664 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67433afd: reopening flushed file at 1730989008668 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46026bdf: reopening flushed file at 1730989008671 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 57f705175086737d129131015f71452d in 1661ms, sequenceid=347, compaction requested=true at 1730989008675 (+4 ms)Writing region close event to WAL at 1730989008731 (+56 ms)Running coprocessor post-close hooks at 1730989008734 (+3 ms)Closed at 1730989008734 2024-11-07T14:16:48,736 INFO [RS_CLOSE_REGION-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(157): Closed 57f705175086737d129131015f71452d 2024-11-07T14:16:48,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=180 updating hbase:meta row=57f705175086737d129131015f71452d, regionState=CLOSED 2024-11-07T14:16:48,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=181, ppid=180, state=RUNNABLE, hasLock=false; CloseRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 because future has completed 2024-11-07T14:16:48,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-07T14:16:48,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; CloseRegionProcedure 57f705175086737d129131015f71452d, server=b7cc9d764de6,37737,1730988826985 in 1.8780 sec 2024-11-07T14:16:48,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=180, resume processing ppid=179 2024-11-07T14:16:48,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, ppid=179, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57f705175086737d129131015f71452d, UNASSIGN in 1.8840 sec 2024-11-07T14:16:48,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-07T14:16:48,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8870 sec 2024-11-07T14:16:48,746 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730989008746"}]},"ts":"1730989008746"} 2024-11-07T14:16:48,748 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T14:16:48,748 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T14:16:48,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9010 sec 2024-11-07T14:16:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-07T14:16:49,003 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:49,003 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-07T14:16:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:49,005 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=182, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-07T14:16:49,005 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=182, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:49,007 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d 2024-11-07T14:16:49,009 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C, FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/recovered.edits] 2024-11-07T14:16:49,011 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/111d251d97f54783bfd9a75318b839d9 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/111d251d97f54783bfd9a75318b839d9 2024-11-07T14:16:49,012 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/47de55aad3f24576ae1f4255ad7ea01b to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/47de55aad3f24576ae1f4255ad7ea01b 2024-11-07T14:16:49,013 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/8264b0e716ca40628632ce60af9e38b3 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/A/8264b0e716ca40628632ce60af9e38b3 2024-11-07T14:16:49,015 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/060e1d83251f40c28302662c551ce20e to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/060e1d83251f40c28302662c551ce20e 2024-11-07T14:16:49,016 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a44a6189e7664394af647945048160a0 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/a44a6189e7664394af647945048160a0 2024-11-07T14:16:49,017 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e1a05270a6fd4288befa3cf94f21cafe to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/B/e1a05270a6fd4288befa3cf94f21cafe 2024-11-07T14:16:49,020 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9e29c8f491144efe9068aba1c5b2601a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/9e29c8f491144efe9068aba1c5b2601a 2024-11-07T14:16:49,024 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b13e8ab4386a43348cc1ce3e12e4cb31 to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/b13e8ab4386a43348cc1ce3e12e4cb31 2024-11-07T14:16:49,025 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/cc6e189006744ed58b15746c176c4f6a to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/C/cc6e189006744ed58b15746c176c4f6a 2024-11-07T14:16:49,028 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/recovered.edits/350.seqid to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d/recovered.edits/350.seqid 2024-11-07T14:16:49,028 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/default/TestAcidGuarantees/57f705175086737d129131015f71452d 2024-11-07T14:16:49,029 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T14:16:49,029 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T14:16:49,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T14:16:49,033 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107011dbcd11c2241a5908c954e9dff41e4_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107011dbcd11c2241a5908c954e9dff41e4_57f705175086737d129131015f71452d 2024-11-07T14:16:49,034 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107251ff5e71ae9472b938005ed83932eeb_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107251ff5e71ae9472b938005ed83932eeb_57f705175086737d129131015f71452d 2024-11-07T14:16:49,035 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072e79323087044a3b9912fd891264cfd7_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072e79323087044a3b9912fd891264cfd7_57f705175086737d129131015f71452d 2024-11-07T14:16:49,037 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107317974be12b0434eb4c98eac7f4b9a57_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107317974be12b0434eb4c98eac7f4b9a57_57f705175086737d129131015f71452d 2024-11-07T14:16:49,038 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073cd417edb9864d998525a27444e0ad85_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073cd417edb9864d998525a27444e0ad85_57f705175086737d129131015f71452d 2024-11-07T14:16:49,039 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074a27fa0e149e491ea6ae9bc0a20a70be_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074a27fa0e149e491ea6ae9bc0a20a70be_57f705175086737d129131015f71452d 2024-11-07T14:16:49,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075570156bfe0a4d50ad8d293f6324c98c_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075570156bfe0a4d50ad8d293f6324c98c_57f705175086737d129131015f71452d 2024-11-07T14:16:49,042 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d08b4ff2c4c4aa2aee4a737052100e2_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d08b4ff2c4c4aa2aee4a737052100e2_57f705175086737d129131015f71452d 2024-11-07T14:16:49,045 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110770f0a34af90b4dcd8f4665f33ebe2ccc_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110770f0a34af90b4dcd8f4665f33ebe2ccc_57f705175086737d129131015f71452d 2024-11-07T14:16:49,046 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774029fd1f8c4455b9f604fead37e93fd_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774029fd1f8c4455b9f604fead37e93fd_57f705175086737d129131015f71452d 2024-11-07T14:16:49,048 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110778b8cdb3b8294e9eb1fcd561aae10f49_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110778b8cdb3b8294e9eb1fcd561aae10f49_57f705175086737d129131015f71452d 2024-11-07T14:16:49,049 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079be6e22a1f5042a68793ec3d55ae330b_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079be6e22a1f5042a68793ec3d55ae330b_57f705175086737d129131015f71452d 2024-11-07T14:16:49,050 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b035dd8d1cd44bc1bf93883721ca100b_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b035dd8d1cd44bc1bf93883721ca100b_57f705175086737d129131015f71452d 2024-11-07T14:16:49,052 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bb5238333ad0481ca7815dce152d9976_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bb5238333ad0481ca7815dce152d9976_57f705175086737d129131015f71452d 2024-11-07T14:16:49,053 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca49b0de296347ae92f3ff0c04a56bcd_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca49b0de296347ae92f3ff0c04a56bcd_57f705175086737d129131015f71452d 2024-11-07T14:16:49,054 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ddb50898ecae47f98792c4a6db895072_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ddb50898ecae47f98792c4a6db895072_57f705175086737d129131015f71452d 2024-11-07T14:16:49,056 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f96c696b7e5b46e691ac086864867351_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f96c696b7e5b46e691ac086864867351_57f705175086737d129131015f71452d 2024-11-07T14:16:49,056 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fe5025c06e4849c8ac62c3ee684e6856_57f705175086737d129131015f71452d to hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fe5025c06e4849c8ac62c3ee684e6856_57f705175086737d129131015f71452d 2024-11-07T14:16:49,057 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T14:16:49,059 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=182, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:49,062 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T14:16:49,067 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T14:16:49,068 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=182, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:49,068 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T14:16:49,068 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730989009068"}]},"ts":"9223372036854775807"} 2024-11-07T14:16:49,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T14:16:49,072 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 57f705175086737d129131015f71452d, NAME => 'TestAcidGuarantees,,1730988975309.57f705175086737d129131015f71452d.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T14:16:49,072 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T14:16:49,072 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730989009072"}]},"ts":"9223372036854775807"} 2024-11-07T14:16:49,077 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T14:16:49,078 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=182, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T14:16:49,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 75 msec 2024-11-07T14:16:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-07T14:16:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T14:16:49,113 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T14:16:49,124 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 243), OpenFileDescriptor=441 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=565 (was 590), ProcessCount=11 (was 11), AvailableMemoryMB=1805 (was 293) - AvailableMemoryMB LEAK? - 2024-11-07T14:16:49,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T14:16:49,125 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T14:16:49,125 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:16:49,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:49,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:49,125 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T14:16:49,125 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-07T14:16:49,126 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b7cc9d764de6,40745,1730988826115 2024-11-07T14:16:49,130 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T14:16:49,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T14:16:49,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T14:16:49,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:16:49,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:16:49,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T14:16:49,161 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T14:16:49,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T14:16:49,161 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:16:49,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:49,162 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b7cc9d764de6,37737,1730988826985' ***** 2024-11-07T14:16:49,162 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T14:16:49,162 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T14:16:49,163 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T14:16:49,163 INFO [RS:0;b7cc9d764de6:37737 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T14:16:49,163 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T14:16:49,163 INFO [RS:0;b7cc9d764de6:37737 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T14:16:49,163 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(959): stopping server b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:49,163 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T14:16:49,163 INFO [RS:0;b7cc9d764de6:37737 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b7cc9d764de6:37737. 2024-11-07T14:16:49,163 DEBUG [RS:0;b7cc9d764de6:37737 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T14:16:49,163 DEBUG [RS:0;b7cc9d764de6:37737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:49,164 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T14:16:49,164 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T14:16:49,164 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T14:16:49,164 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T14:16:49,165 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T14:16:49,165 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-07T14:16:49,165 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T14:16:49,165 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T14:16:49,165 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T14:16:49,165 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T14:16:49,165 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T14:16:49,165 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T14:16:49,165 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-07T14:16:49,171 INFO [regionserver/b7cc9d764de6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T14:16:49,196 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/info/c6dc5f9f0a184d8ca06f8069e64ab48b is 95, key is TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd./info:/1730988861194/DeleteFamily/seqid=0 2024-11-07T14:16:49,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742491_1667 (size=6514) 2024-11-07T14:16:49,365 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T14:16:49,565 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T14:16:49,601 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/info/c6dc5f9f0a184d8ca06f8069e64ab48b 2024-11-07T14:16:49,622 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/ns/42ab692bde1a4587aece5673c02c6eff is 93, key is TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd./ns:/1730988861103/DeleteFamily/seqid=0 2024-11-07T14:16:49,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742492_1668 (size=5906) 2024-11-07T14:16:49,766 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T14:16:49,966 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T14:16:49,993 INFO [regionserver/b7cc9d764de6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T14:16:49,993 INFO [regionserver/b7cc9d764de6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T14:16:50,030 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/ns/42ab692bde1a4587aece5673c02c6eff 2024-11-07T14:16:50,055 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/rep_barrier/19e799117a3143649db1f9269d13f14d is 102, key is TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd./rep_barrier:/1730988861103/DeleteFamily/seqid=0 2024-11-07T14:16:50,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742493_1669 (size=6025) 2024-11-07T14:16:50,166 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T14:16:50,366 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T14:16:50,460 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/rep_barrier/19e799117a3143649db1f9269d13f14d 2024-11-07T14:16:50,499 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/table/4df55ad2d3f84acf8230b0887c11afa2 is 96, key is TestAcidGuarantees,,1730988830520.8133e138f01b5d4b631e25ead6cfe8cd./table:/1730988861103/DeleteFamily/seqid=0 2024-11-07T14:16:50,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742494_1670 (size=5936) 2024-11-07T14:16:50,531 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/table/4df55ad2d3f84acf8230b0887c11afa2 2024-11-07T14:16:50,537 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/info/c6dc5f9f0a184d8ca06f8069e64ab48b as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/info/c6dc5f9f0a184d8ca06f8069e64ab48b 2024-11-07T14:16:50,548 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/info/c6dc5f9f0a184d8ca06f8069e64ab48b, entries=12, sequenceid=90, filesize=6.4 K 2024-11-07T14:16:50,550 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/ns/42ab692bde1a4587aece5673c02c6eff as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/ns/42ab692bde1a4587aece5673c02c6eff 2024-11-07T14:16:50,554 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/ns/42ab692bde1a4587aece5673c02c6eff, entries=8, sequenceid=90, filesize=5.8 K 2024-11-07T14:16:50,555 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/rep_barrier/19e799117a3143649db1f9269d13f14d as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/rep_barrier/19e799117a3143649db1f9269d13f14d 2024-11-07T14:16:50,564 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/rep_barrier/19e799117a3143649db1f9269d13f14d, entries=6, sequenceid=90, filesize=5.9 K 2024-11-07T14:16:50,567 DEBUG [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T14:16:50,568 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/.tmp/table/4df55ad2d3f84acf8230b0887c11afa2 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/table/4df55ad2d3f84acf8230b0887c11afa2 2024-11-07T14:16:50,571 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/table/4df55ad2d3f84acf8230b0887c11afa2, entries=7, sequenceid=90, filesize=5.8 K 2024-11-07T14:16:50,572 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1407ms, sequenceid=90, compaction requested=false 2024-11-07T14:16:50,578 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-07T14:16:50,579 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T14:16:50,579 INFO [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T14:16:50,579 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730989009165Running coprocessor pre-close hooks at 1730989009165Disabling compacts and flushes for region at 1730989009165Disabling writes for close at 1730989009165Obtaining lock to block concurrent updates at 1730989009165Preparing flush snapshotting stores in 1588230740 at 1730989009165Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1730989009166 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730989009167 (+1 ms)Flushing 1588230740/info: creating writer at 1730989009167Flushing 1588230740/info: appending metadata at 1730989009196 (+29 ms)Flushing 1588230740/info: closing flushed file at 1730989009196Flushing 1588230740/ns: creating writer at 1730989009605 (+409 ms)Flushing 1588230740/ns: appending metadata at 1730989009622 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1730989009622Flushing 1588230740/rep_barrier: creating writer at 1730989010035 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1730989010055 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1730989010055Flushing 1588230740/table: creating writer at 1730989010465 (+410 ms)Flushing 1588230740/table: appending metadata at 1730989010498 (+33 ms)Flushing 1588230740/table: closing flushed file at 1730989010498Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e073cbe: reopening flushed file at 1730989010536 (+38 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7519d75e: reopening flushed file at 1730989010548 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@694f3f26: reopening flushed file at 1730989010554 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2497b127: reopening flushed file at 1730989010564 (+10 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1407ms, sequenceid=90, compaction requested=false at 1730989010572 (+8 ms)Writing region close event to WAL at 1730989010575 (+3 ms)Running coprocessor post-close hooks at 1730989010579 (+4 ms)Closed at 1730989010579 2024-11-07T14:16:50,579 DEBUG [RS_CLOSE_META-regionserver/b7cc9d764de6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T14:16:50,767 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(976): stopping server b7cc9d764de6,37737,1730988826985; all regions closed. 2024-11-07T14:16:50,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741834_1010 (size=25344) 2024-11-07T14:16:50,785 DEBUG [RS:0;b7cc9d764de6:37737 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/oldWALs 2024-11-07T14:16:50,785 INFO [RS:0;b7cc9d764de6:37737 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b7cc9d764de6%2C37737%2C1730988826985.meta:.meta(num 1730988830038) 2024-11-07T14:16:50,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741832_1008 (size=15462099) 2024-11-07T14:16:50,800 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/WALs/b7cc9d764de6,37737,1730988826985/b7cc9d764de6%2C37737%2C1730988826985.1730988829065 not finished, retry = 0 2024-11-07T14:16:50,903 DEBUG [RS:0;b7cc9d764de6:37737 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/oldWALs 2024-11-07T14:16:50,904 INFO [RS:0;b7cc9d764de6:37737 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b7cc9d764de6%2C37737%2C1730988826985:(num 1730988829065) 2024-11-07T14:16:50,904 DEBUG [RS:0;b7cc9d764de6:37737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T14:16:50,904 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T14:16:50,904 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T14:16:50,904 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.ChoreService(370): Chore service for: regionserver/b7cc9d764de6:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T14:16:50,904 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T14:16:50,905 INFO [RS:0;b7cc9d764de6:37737 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37737 2024-11-07T14:16:50,906 INFO [regionserver/b7cc9d764de6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T14:16:50,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T14:16:50,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b7cc9d764de6,37737,1730988826985 2024-11-07T14:16:50,919 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T14:16:50,920 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b7cc9d764de6,37737,1730988826985] 2024-11-07T14:16:50,938 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b7cc9d764de6,37737,1730988826985 already deleted, retry=false 2024-11-07T14:16:50,939 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b7cc9d764de6,37737,1730988826985 expired; onlineServers=0 2024-11-07T14:16:50,939 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b7cc9d764de6,40745,1730988826115' ***** 2024-11-07T14:16:50,939 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T14:16:50,939 INFO [M:0;b7cc9d764de6:40745 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T14:16:50,939 INFO [M:0;b7cc9d764de6:40745 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T14:16:50,939 DEBUG [M:0;b7cc9d764de6:40745 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T14:16:50,939 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T14:16:50,940 DEBUG [M:0;b7cc9d764de6:40745 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T14:16:50,940 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster-HFileCleaner.large.0-1730988828748 {}] cleaner.HFileCleaner(306): Exit Thread[master/b7cc9d764de6:0:becomeActiveMaster-HFileCleaner.large.0-1730988828748,5,FailOnTimeoutGroup] 2024-11-07T14:16:50,940 INFO [M:0;b7cc9d764de6:40745 {}] hbase.ChoreService(370): Chore service for: master/b7cc9d764de6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T14:16:50,940 INFO [M:0;b7cc9d764de6:40745 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T14:16:50,940 DEBUG [M:0;b7cc9d764de6:40745 {}] master.HMaster(1795): Stopping service threads 2024-11-07T14:16:50,940 INFO [M:0;b7cc9d764de6:40745 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T14:16:50,940 INFO [M:0;b7cc9d764de6:40745 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T14:16:50,940 ERROR [M:0;b7cc9d764de6:40745 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-07T14:16:50,941 INFO [M:0;b7cc9d764de6:40745 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T14:16:50,941 DEBUG [master/b7cc9d764de6:0:becomeActiveMaster-HFileCleaner.small.0-1730988828754 {}] cleaner.HFileCleaner(306): Exit Thread[master/b7cc9d764de6:0:becomeActiveMaster-HFileCleaner.small.0-1730988828754,5,FailOnTimeoutGroup] 2024-11-07T14:16:50,942 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T14:16:50,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T14:16:50,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T14:16:50,952 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-07T14:16:50,953 INFO [M:0;b7cc9d764de6:40745 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/.lastflushedseqids 2024-11-07T14:16:50,953 DEBUG [RegionServerTracker-0 {}] master.ActiveMasterManager(353): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-07T14:16:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742495_1671 (size=72) 2024-11-07T14:16:51,032 INFO [RS:0;b7cc9d764de6:37737 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T14:16:51,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T14:16:51,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37737-0x10116d31ff60001, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T14:16:51,032 INFO [RS:0;b7cc9d764de6:37737 {}] regionserver.HRegionServer(1031): Exiting; stopping=b7cc9d764de6,37737,1730988826985; zookeeper connection closed. 2024-11-07T14:16:51,032 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@656e31a3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@656e31a3 2024-11-07T14:16:51,033 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T14:16:51,376 INFO [M:0;b7cc9d764de6:40745 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T14:16:51,377 INFO [M:0;b7cc9d764de6:40745 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T14:16:51,377 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T14:16:51,377 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:16:51,377 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:16:51,377 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T14:16:51,377 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:16:51,377 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=772.93 KB heapSize=951.47 KB 2024-11-07T14:16:51,393 DEBUG [M:0;b7cc9d764de6:40745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb71c1913968459bbeefb9b3c86b8561 is 82, key is hbase:meta,,1/info:regioninfo/1730988830125/Put/seqid=0 2024-11-07T14:16:51,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742496_1672 (size=5672) 2024-11-07T14:16:51,797 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2199 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb71c1913968459bbeefb9b3c86b8561 2024-11-07T14:16:51,824 DEBUG [M:0;b7cc9d764de6:40745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/da2e56a1f11040e1b3edce9007d40a9e is 2275, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1730988864541/Put/seqid=0 2024-11-07T14:16:51,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742497_1673 (size=46209) 2024-11-07T14:16:52,228 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=772.37 KB at sequenceid=2199 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/da2e56a1f11040e1b3edce9007d40a9e 2024-11-07T14:16:52,232 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for da2e56a1f11040e1b3edce9007d40a9e 2024-11-07T14:16:52,249 DEBUG [M:0;b7cc9d764de6:40745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e38793d7245348a4b4e3bce245ca8735 is 69, key is b7cc9d764de6,37737,1730988826985/rs:state/1730988828803/Put/seqid=0 2024-11-07T14:16:52,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073742498_1674 (size=5156) 2024-11-07T14:16:52,652 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2199 (bloomFilter=true), to=hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e38793d7245348a4b4e3bce245ca8735 2024-11-07T14:16:52,658 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb71c1913968459bbeefb9b3c86b8561 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eb71c1913968459bbeefb9b3c86b8561 2024-11-07T14:16:52,663 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eb71c1913968459bbeefb9b3c86b8561, entries=8, sequenceid=2199, filesize=5.5 K 2024-11-07T14:16:52,664 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/da2e56a1f11040e1b3edce9007d40a9e as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/da2e56a1f11040e1b3edce9007d40a9e 2024-11-07T14:16:52,669 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for da2e56a1f11040e1b3edce9007d40a9e 2024-11-07T14:16:52,669 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/da2e56a1f11040e1b3edce9007d40a9e, entries=182, sequenceid=2199, filesize=45.1 K 2024-11-07T14:16:52,670 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e38793d7245348a4b4e3bce245ca8735 as hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e38793d7245348a4b4e3bce245ca8735 2024-11-07T14:16:52,675 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43161/user/jenkins/test-data/50604b8f-b376-606c-a9bb-77628e4dc4f9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e38793d7245348a4b4e3bce245ca8735, entries=1, sequenceid=2199, filesize=5.0 K 2024-11-07T14:16:52,676 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(3140): Finished flush of dataSize ~772.93 KB/791480, heapSize ~951.17 KB/974000, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2199, compaction requested=false 2024-11-07T14:16:52,678 INFO [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T14:16:52,678 DEBUG [M:0;b7cc9d764de6:40745 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730989011377Disabling compacts and flushes for region at 1730989011377Disabling writes for close at 1730989011377Obtaining lock to block concurrent updates at 1730989011377Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730989011377Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=791480, getHeapSize=974240, getOffHeapSize=0, getCellsCount=2406 at 1730989011377Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730989011378 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730989011378Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730989011392 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730989011392Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730989011805 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730989011824 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730989011824Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730989012232 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730989012248 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730989012248Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79357607: reopening flushed file at 1730989012657 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ba0fc9: reopening flushed file at 1730989012663 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ef84d2b: reopening flushed file at 1730989012669 (+6 ms)Finished flush of dataSize ~772.93 KB/791480, heapSize ~951.17 KB/974000, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2199, compaction requested=false at 1730989012676 (+7 ms)Writing region close event to WAL at 1730989012678 (+2 ms)Closed at 1730989012678 2024-11-07T14:16:52,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41543 is added to blk_1073741830_1006 (size=935662) 2024-11-07T14:16:52,683 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T14:16:52,683 INFO [M:0;b7cc9d764de6:40745 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T14:16:52,683 INFO [M:0;b7cc9d764de6:40745 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40745 2024-11-07T14:16:52,683 INFO [M:0;b7cc9d764de6:40745 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T14:16:52,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T14:16:52,797 INFO [M:0;b7cc9d764de6:40745 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T14:16:52,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40745-0x10116d31ff60000, quorum=127.0.0.1:50891, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T14:16:52,806 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T14:16:52,809 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T14:16:52,809 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T14:16:52,809 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T14:16:52,810 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/hadoop.log.dir/,STOPPED} 2024-11-07T14:16:52,812 WARN [BP-900104924-172.17.0.3-1730988822009 heartbeating to localhost/127.0.0.1:43161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T14:16:52,812 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T14:16:52,813 WARN [BP-900104924-172.17.0.3-1730988822009 heartbeating to localhost/127.0.0.1:43161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-900104924-172.17.0.3-1730988822009 (Datanode Uuid 73886870-0704-415d-af4d-78dea2de931a) service to localhost/127.0.0.1:43161 2024-11-07T14:16:52,813 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T14:16:52,816 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/data/data1/current/BP-900104924-172.17.0.3-1730988822009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T14:16:52,816 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/cluster_e1210853-18a4-8729-d28a-6f423f1c82bf/data/data2/current/BP-900104924-172.17.0.3-1730988822009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T14:16:52,817 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T14:16:52,823 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T14:16:52,824 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T14:16:52,824 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T14:16:52,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T14:16:52,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1cf01364-aa80-62f9-8f29-9ab7d6b2f2d5/hadoop.log.dir/,STOPPED} 2024-11-07T14:16:52,840 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T14:16:52,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down